{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "%env LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
    "%env LLM_API_KEY=sk-替换为自己的Qwen API Key，评估用\n",
    "%env TAVILY_API_KEY=tvly-替换为自己的Tavily API Key，检索外部知识用"
   ],
   "id": "a8bdf300749ab170"
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "af375836-b870-458b-87d1-4e00565977eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:20:46.284653Z",
     "iopub.status.busy": "2024-11-27T11:20:46.283895Z",
     "iopub.status.idle": "2024-11-27T11:20:46.289381Z",
     "shell.execute_reply": "2024-11-27T11:20:46.289076Z",
     "shell.execute_reply.started": "2024-11-27T11:20:46.284574Z"
    },
    "papermill": {
     "duration": 0.115454,
     "end_time": "2024-11-23T14:29:00.919641",
     "exception": false,
     "start_time": "2024-11-23T14:29:00.804187",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%capture --no-stderr\n",
    "!pip install -U langchain langchain-community langchain-openai langchain_chroma pypdf sentence_transformers chromadb shutil openpyxl FlagEmbedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1e2c72b8-ee12-4130-af88-699998aa230c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:20:46.289949Z",
     "iopub.status.busy": "2024-11-27T11:20:46.289832Z",
     "iopub.status.idle": "2024-11-27T11:20:46.523751Z",
     "shell.execute_reply": "2024-11-27T11:20:46.523253Z",
     "shell.execute_reply.started": "2024-11-27T11:20:46.289937Z"
    },
    "papermill": {
     "duration": 0.319981,
     "end_time": "2024-11-23T14:29:01.380771",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.060790",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "841d2b02-ad06-40d2-b11f-c7adccec6ca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:20:46.525166Z",
     "iopub.status.busy": "2024-11-27T11:20:46.524960Z",
     "iopub.status.idle": "2024-11-27T11:20:46.527971Z",
     "shell.execute_reply": "2024-11-27T11:20:46.527595Z",
     "shell.execute_reply.started": "2024-11-27T11:20:46.525153Z"
    },
    "papermill": {
     "duration": 0.121409,
     "end_time": "2024-11-23T14:29:01.638126",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.516717",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "expr_version = 'retrieval_v14_crag'\n",
    "\n",
    "preprocess_output_dir = os.path.join(os.path.pardir, 'outputs', 'v1_20240713')\n",
    "expr_dir = os.path.join(os.path.pardir, 'experiments', expr_version)\n",
    "\n",
    "os.makedirs(expr_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf7e81e3-4c82-4842-aef5-7592caaf1d39",
   "metadata": {
    "papermill": {
     "duration": 0.100379,
     "end_time": "2024-11-23T14:29:01.862379",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.762000",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 读取文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e6920e29-bc7d-4635-be06-d151eaf0e100",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:20:46.528567Z",
     "iopub.status.busy": "2024-11-27T11:20:46.528424Z",
     "iopub.status.idle": "2024-11-27T11:20:48.274425Z",
     "shell.execute_reply": "2024-11-27T11:20:48.273967Z",
     "shell.execute_reply.started": "2024-11-27T11:20:46.528555Z"
    },
    "papermill": {
     "duration": 2.012298,
     "end_time": "2024-11-23T14:29:03.974974",
     "exception": false,
     "start_time": "2024-11-23T14:29:01.962676",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "\n",
    "loader = PyPDFLoader(os.path.join(os.path.pardir, 'data', '2024全球经济金融展望报告.pdf'))\n",
    "documents = loader.load()\n",
    "\n",
    "qa_df = pd.read_excel(os.path.join(preprocess_output_dir, 'question_answer.xlsx'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "841ec659-4ad7-4e1f-b1ea-3477bf97fde3",
   "metadata": {
    "papermill": {
     "duration": 0.100297,
     "end_time": "2024-11-23T14:29:04.219302",
     "exception": false,
     "start_time": "2024-11-23T14:29:04.119005",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 文档切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "74fe856a-7c19-4c3c-bb30-7abfa6298f74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:20:48.275380Z",
     "iopub.status.busy": "2024-11-27T11:20:48.274994Z",
     "iopub.status.idle": "2024-11-27T11:20:48.282448Z",
     "shell.execute_reply": "2024-11-27T11:20:48.282031Z",
     "shell.execute_reply.started": "2024-11-27T11:20:48.275366Z"
    },
    "papermill": {
     "duration": 0.109229,
     "end_time": "2024-11-23T14:29:04.429069",
     "exception": false,
     "start_time": "2024-11-23T14:29:04.319840",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from uuid import uuid4\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "def split_docs(documents, filepath, chunk_size=400, chunk_overlap=40, seperators=['\\n\\n\\n', '\\n\\n'], force_split=False):\n",
    "    if os.path.exists(filepath) and not force_split:\n",
    "        print('found cache, restoring...')\n",
    "        return pickle.load(open(filepath, 'rb'))\n",
    "\n",
    "    splitter = RecursiveCharacterTextSplitter(\n",
    "        chunk_size=chunk_size,\n",
    "        chunk_overlap=chunk_overlap,\n",
    "        separators=seperators\n",
    "    )\n",
    "    split_docs = splitter.split_documents(documents)\n",
    "    for chunk in split_docs:\n",
    "        chunk.metadata['uuid'] = str(uuid4())\n",
    "\n",
    "    pickle.dump(split_docs, open(filepath, 'wb'))\n",
    "\n",
    "    return split_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aa25540d-0504-4ae7-9804-9e3862b132d5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:20:48.283136Z",
     "iopub.status.busy": "2024-11-27T11:20:48.282948Z",
     "iopub.status.idle": "2024-11-27T11:20:48.290791Z",
     "shell.execute_reply": "2024-11-27T11:20:48.290395Z",
     "shell.execute_reply.started": "2024-11-27T11:20:48.283124Z"
    },
    "papermill": {
     "duration": 0.145583,
     "end_time": "2024-11-23T14:29:04.677429",
     "exception": false,
     "start_time": "2024-11-23T14:29:04.531846",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found cache, restoring...\n"
     ]
    }
   ],
   "source": [
    "splitted_docs = split_docs(documents, os.path.join(preprocess_output_dir, 'split_docs.pkl'), chunk_size=500, chunk_overlap=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "220dbc3a-fceb-4e49-a3f1-01e16660b2a6",
   "metadata": {
    "papermill": {
     "duration": 0.100209,
     "end_time": "2024-11-23T14:29:05.255871",
     "exception": false,
     "start_time": "2024-11-23T14:29:05.155662",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 为知识库构建索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8598a11c-25d8-4af1-a98b-06a8c394e261",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:20:48.291733Z",
     "iopub.status.busy": "2024-11-27T11:20:48.291331Z",
     "iopub.status.idle": "2024-11-27T11:20:49.125311Z",
     "shell.execute_reply": "2024-11-27T11:20:49.124846Z",
     "shell.execute_reply.started": "2024-11-27T11:20:48.291719Z"
    },
    "papermill": {
     "duration": 0.989203,
     "end_time": "2024-11-23T14:29:06.345534",
     "exception": false,
     "start_time": "2024-11-23T14:29:05.356331",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device: cuda\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "import torch\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(f'device: {device}')\n",
    "\n",
    "def get_embeddings(model_path):\n",
    "    embeddings = HuggingFaceBgeEmbeddings(\n",
    "        model_name=model_path,\n",
    "        model_kwargs={'device': device},\n",
    "        encode_kwargs={'normalize_embeddings': True},\n",
    "        # show_progress=True\n",
    "        query_instruction='为这个句子生成表示以用于检索相关文章：'\n",
    "    )\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f6f46c73-7369-448f-a89a-ed3d817cad47",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:20:49.126021Z",
     "iopub.status.busy": "2024-11-27T11:20:49.125812Z",
     "iopub.status.idle": "2024-11-27T11:21:09.270304Z",
     "shell.execute_reply": "2024-11-27T11:21:09.267845Z",
     "shell.execute_reply.started": "2024-11-27T11:20:49.126008Z"
    },
    "papermill": {
     "duration": 83.983138,
     "end_time": "2024-11-23T14:35:06.117207",
     "exception": false,
     "start_time": "2024-11-23T14:33:42.134069",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import shutil\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "from langchain_community.vectorstores import Chroma\n",
    "\n",
    "model_path = 'BAAI/bge-large-zh-v1.5'\n",
    "embeddings = get_embeddings(model_path)\n",
    "\n",
    "persist_directory = os.path.join(expr_dir, 'chroma', 'bge')\n",
    "shutil.rmtree(persist_directory, ignore_errors=True)\n",
    "vector_db = Chroma.from_documents(\n",
    "    splitted_docs,\n",
    "    embedding=embeddings,\n",
    "    persist_directory=persist_directory\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbc9851f-52c1-441f-98c4-afde02715c40",
   "metadata": {},
   "source": [
    "# CRAG"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fba51e1e-1cae-4498-b2a1-323f4e4d45a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-26T15:33:34.670303Z",
     "iopub.status.busy": "2024-11-26T15:33:34.669874Z",
     "iopub.status.idle": "2024-11-26T15:33:34.674898Z",
     "shell.execute_reply": "2024-11-26T15:33:34.673981Z",
     "shell.execute_reply.started": "2024-11-26T15:33:34.670265Z"
    }
   },
   "source": [
    "## State定义"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e825ff6d-e9bf-44ad-8b1a-be79aaa59ace",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:09.274361Z",
     "iopub.status.busy": "2024-11-27T11:21:09.273202Z",
     "iopub.status.idle": "2024-11-27T11:21:09.286420Z",
     "shell.execute_reply": "2024-11-27T11:21:09.284056Z",
     "shell.execute_reply.started": "2024-11-27T11:21:09.274285Z"
    }
   },
   "outputs": [],
   "source": [
    "from typing import Dict, TypedDict\n",
    "\n",
    "class GraphState(TypedDict):\n",
    "    \"\"\"\n",
    "    Represents the state of an agent in the conversation.\n",
    "\n",
    "    Attributes:\n",
    "        keys: A dictionary where each key is a string and the value is expected to be a list or another structure\n",
    "              that supports addition with `operator.add`. This could be used, for instance, to accumulate messages\n",
    "              or other pieces of data throughout the graph.\n",
    "    \"\"\"\n",
    "\n",
    "    keys: Dict[str, any]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "363b070d-7991-4ab8-8d14-65f22a073cac",
   "metadata": {},
   "source": [
    "## Nodes和Edges定义"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "845d9b99-2908-476d-a323-9ada5aadf211",
   "metadata": {},
   "source": [
    "### Nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "91fb3477-d8f1-4d15-823c-095f48eefc6b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:09.289815Z",
     "iopub.status.busy": "2024-11-27T11:21:09.289089Z",
     "iopub.status.idle": "2024-11-27T11:21:09.571458Z",
     "shell.execute_reply": "2024-11-27T11:21:09.571011Z",
     "shell.execute_reply.started": "2024-11-27T11:21:09.289747Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py:3460: LangChainDeprecationWarning: As of langchain-core 0.3.0, LangChain uses pydantic v2 internally. The langchain_core.pydantic_v1 module was a compatibility shim for pydantic v1, and should no longer be used. Please update the code to import from Pydantic directly.\n",
      "\n",
      "For example, replace imports like: `from langchain_core.pydantic_v1 import BaseModel`\n",
      "with: `from pydantic import BaseModel`\n",
      "or the v1 compatibility namespace if you are working in a code base that has not been fully upgraded to pydantic 2 yet. \tfrom pydantic.v1 import BaseModel\n",
      "\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "from langchain.output_parsers.openai_tools import PydanticToolsParser\n",
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.schema import Document\n",
    "from langchain_community.tools.tavily_search import TavilySearchResults\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_core.pydantic_v1 import BaseModel, Field\n",
    "from langchain_core.utils.function_calling import convert_to_openai_tool\n",
    "from langchain.llms import Ollama\n",
    "\n",
    "def retrieve(state):\n",
    "    \"\"\"\n",
    "    Retrieve documents\n",
    "\n",
    "    Args:\n",
    "        state (dict): The current state of the agent, including all keys.\n",
    "\n",
    "    Returns:\n",
    "        dict: New key added to state, documents, that contains documents.\n",
    "    \"\"\"\n",
    "    print(\"---RETRIEVE---\")\n",
    "    state_dict = state[\"keys\"]\n",
    "    question = state_dict[\"question\"]\n",
    "    retriever = vector_db.as_retriever(search_kwargs={'k': 3})\n",
    "    documents = retriever.invoke(question)\n",
    "    return {\"keys\": {\"documents\": documents, \"question\": question}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "dfa74204-a559-4055-9355-ab08a002f72e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:30:27.411905Z",
     "iopub.status.busy": "2024-11-27T11:30:27.411732Z",
     "iopub.status.idle": "2024-11-27T11:30:27.415767Z",
     "shell.execute_reply": "2024-11-27T11:30:27.415435Z",
     "shell.execute_reply.started": "2024-11-27T11:30:27.411891Z"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def rag(query, llm, documents=None, n_chunks=3):\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个金融分析师，擅长根据所获取的信息片段，对问题进行分析和推理。\n",
    "你的任务是根据所获取的信息片段（<<<<context>>><<<</context>>>之间的内容）回答问题。\n",
    "回答保持简洁，不必重复问题，不要添加描述性解释和与答案无关的任何内容。\n",
    "已知信息：\n",
    "<<<<context>>>\n",
    "{{knowledge}}\n",
    "<<<</context>>>\n",
    "\n",
    "问题：{{query}}\n",
    "请回答：\n",
    "\"\"\".strip()\n",
    "    if documents is None:\n",
    "        chunks = vector_db.similarity_search(query, k=n_chunks)\n",
    "    else:\n",
    "        chunks = documents\n",
    "        \n",
    "    prompt = prompt_tmpl.replace('{{knowledge}}', '\\n\\n'.join([doc.page_content for doc in chunks])).replace('{{query}}', query)\n",
    "    retry_count = 3\n",
    "\n",
    "    resp = ''\n",
    "    while retry_count > 0:\n",
    "        try:\n",
    "            resp = llm.invoke(prompt)\n",
    "            break\n",
    "        except Exception as e:\n",
    "            retry_count -= 1\n",
    "            sleeping_seconds = 2 ** (4 - retry_count)\n",
    "            print(f\"query={query}, error={e}, sleeping={sleeping_seconds}, remaining retry count={retry_count}\")\n",
    "            \n",
    "            time.sleep(sleeping_seconds)\n",
    "\n",
    "    if not isinstance(resp, str):\n",
    "        resp = resp.content\n",
    "    \n",
    "    return resp, chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f25877e2-e7a7-49fa-af5d-7dc5d80ac42a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:09.577546Z",
     "iopub.status.busy": "2024-11-27T11:21:09.577398Z",
     "iopub.status.idle": "2024-11-27T11:21:09.590269Z",
     "shell.execute_reply": "2024-11-27T11:21:09.589758Z",
     "shell.execute_reply.started": "2024-11-27T11:21:09.577535Z"
    }
   },
   "outputs": [],
   "source": [
    "def generate(state):\n",
    "    \"\"\"\n",
    "    Generate answer\n",
    "\n",
    "    Args:\n",
    "        state (dict): The current state of the agent, including all keys.\n",
    "\n",
    "    Returns:\n",
    "        dict: New key added to state, generation, that contains generation.\n",
    "    \"\"\"\n",
    "    print(\"---GENERATE---\")\n",
    "    state_dict = state[\"keys\"]\n",
    "    question = state_dict[\"question\"]\n",
    "    documents = state_dict[\"documents\"]\n",
    "\n",
    "    # LLM\n",
    "    # llm = ChatOpenAI(model=\"gpt-3.5-turbo\", temperature=0, streaming=True)\n",
    "    llm = Ollama(\n",
    "        model='qwen2:7b-instruct',\n",
    "        base_url='http://localhost:11434'\n",
    "    )\n",
    "    generation, documents = rag(question, llm, documents)\n",
    "    return {\n",
    "        \"keys\": {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f9db49fa-332b-442d-941c-a58283413e7d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:09.590971Z",
     "iopub.status.busy": "2024-11-27T11:21:09.590762Z",
     "iopub.status.idle": "2024-11-27T11:21:09.647144Z",
     "shell.execute_reply": "2024-11-27T11:21:09.646606Z",
     "shell.execute_reply.started": "2024-11-27T11:21:09.590959Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_experimental.llms.ollama_functions import OllamaFunctions\n",
    "\n",
    "def grade_documents(state):\n",
    "    \"\"\"\n",
    "    Determines whether the retrieved documents are relevant to the question.\n",
    "\n",
    "    Args:\n",
    "        state (dict): The current state of the agent, including all keys.\n",
    "\n",
    "    Returns:\n",
    "        dict: New key added to state, filtered_documents, that contains relevant documents.\n",
    "    \"\"\"\n",
    "\n",
    "    print(\"---CHECK RELEVANCE---\")\n",
    "    state_dict = state[\"keys\"]\n",
    "    question = state_dict[\"question\"]\n",
    "    documents = state_dict[\"documents\"]\n",
    "\n",
    "    # Data model\n",
    "    class grade(BaseModel):\n",
    "        \"\"\"Binary score for relevance check.\"\"\"\n",
    "\n",
    "        binary_score: str = Field(description=\"Relevance score 'yes' or 'no'\")\n",
    "\n",
    "    # Tool\n",
    "    grade_tool_oai = convert_to_openai_tool(grade)\n",
    "    \n",
    "    # LLM\n",
    "    # model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
    "    # LLM with tool and enforce invocation\n",
    "    # 这个是使用gpt-4模型时的工具绑定方法\n",
    "    # llm_with_tool = model.bind(\n",
    "    #     tools=[convert_to_openai_tool(grade_tool_oai)],\n",
    "    #     tool_choice={\"type\": \"function\", \"function\": {\"name\": \"grade\"}},\n",
    "    # )\n",
    "    \n",
    "    model = OllamaFunctions(\n",
    "        model='qwen2:7b-instruct-32k',\n",
    "        base_url='http://localhost:11434',\n",
    "        format='json'\n",
    "    )\n",
    "    \"\"\"\n",
    "    convert_to_openai_tool(grade_tool_oai)的数据结构如下，但我们只需要function部分\n",
    "    {'type': 'function',\n",
    "     'function': {'name': 'grade',\n",
    "      'description': 'Binary score for relevance check.',\n",
    "      'parameters': {'type': 'object',\n",
    "       'properties': {'binary_score': {'description': \"Relevance score 'yes' or 'no'\",\n",
    "         'type': 'string'}},\n",
    "       'required': ['binary_score']}}}\n",
    "    \"\"\"\n",
    "    llm_with_tool = model.bind_tools(\n",
    "        tools=[convert_to_openai_tool(grade_tool_oai)['function']],\n",
    "        function_call={\"name\": \"grade\"},\n",
    "    )\n",
    "\n",
    "    # Parser\n",
    "    parser_tool = PydanticToolsParser(tools=[grade])\n",
    "\n",
    "    # Prompt\n",
    "    prompt = PromptTemplate(\n",
    "        template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
    "        Here is the retrieved document: \\n\\n {context} \\n\\n\n",
    "        Here is the user question: {question} \\n\n",
    "        If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
    "        Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\",\n",
    "        input_variables=[\"context\", \"question\"],\n",
    "    )\n",
    "\n",
    "    # Chain\n",
    "    chain = prompt | llm_with_tool | parser_tool\n",
    "\n",
    "    # Score\n",
    "    filtered_docs = []\n",
    "    search = \"No\"  # Default do not opt for web search to supplement retrieval\n",
    "    for d in documents:\n",
    "        score = chain.invoke({\"question\": question, \"context\": d.page_content})\n",
    "        grade = score[0].binary_score\n",
    "        if grade == \"yes\":\n",
    "            print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
    "            filtered_docs.append(d)\n",
    "        else:\n",
    "            print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
    "            search = \"Yes\"  # Perform web search\n",
    "            continue\n",
    "\n",
    "    return {\n",
    "        \"keys\": {\n",
    "            \"documents\": filtered_docs,\n",
    "            \"question\": question,\n",
    "            \"run_web_search\": search,\n",
    "        }\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f398d375-c09b-4a41-97b5-caf1da36ebb0",
   "metadata": {},
   "source": [
    "这面这段代码涉及到函数调用，如果执行出错，可以把核心部分提取出来进行调试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dc2712d1-2217-4ffa-9e63-afecabafce73",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:09.647700Z",
     "iopub.status.busy": "2024-11-27T11:21:09.647578Z",
     "iopub.status.idle": "2024-11-27T11:21:09.718905Z",
     "shell.execute_reply": "2024-11-27T11:21:09.718502Z",
     "shell.execute_reply.started": "2024-11-27T11:21:09.647688Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'type': 'function', 'function': {'name': 'grade', 'description': 'Binary score for relevance check.', 'parameters': {'type': 'object', 'properties': {'binary_score': {'description': \"Relevance score 'yes' or 'no'\", 'type': 'string'}}, 'required': ['binary_score']}}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3999598/2245898911.py:3: LangChainDeprecationWarning: The class `OllamaFunctions` was deprecated in LangChain 0.0.64 and will be removed in 1.0. An updated version of the class exists in the :class:`~langchain-ollama package and should be used instead. To use it run `pip install -U :class:`~langchain-ollama` and import as `from :class:`~langchain_ollama import ChatOllama``.\n",
      "  model = OllamaFunctions(\n"
     ]
    }
   ],
   "source": [
    "from langchain_experimental.llms.ollama_functions import OllamaFunctions\n",
    "\n",
    "model = OllamaFunctions(\n",
    "    model='qwen2:7b-instruct-32k',\n",
    "    # model='qwen2.5:7b',\n",
    "    base_url='http://localhost:11434',\n",
    "    format='json'\n",
    ")\n",
    "\n",
    "# 使用OllamaFunctions时会警告建议使用ChatOllama，但ChatOllama不支持函数调用\n",
    "# from langchain_ollama import ChatOllama\n",
    "# model = ChatOllama(\n",
    "#     model='qwen2:7b-instruct-32k',\n",
    "#     base_url='http://localhost:11434',\n",
    "#     format='json'\n",
    "# )\n",
    "\n",
    "class grade(BaseModel):\n",
    "    \"\"\"Binary score for relevance check.\"\"\"\n",
    "\n",
    "    binary_score: str = Field(description=\"Relevance score 'yes' or 'no'\")\n",
    "\n",
    "grade_tool_oai = convert_to_openai_tool(grade)\n",
    "llm_with_tool = model.bind_tools(\n",
    "    tools=[convert_to_openai_tool(grade_tool_oai)['function']],\n",
    "    function_call={\"name\": \"grade\"},\n",
    ")\n",
    "\n",
    "# Parser\n",
    "parser_tool = PydanticToolsParser(tools=[grade])\n",
    "\n",
    "# Prompt\n",
    "prompt = PromptTemplate(\n",
    "    template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
    "    Here is the retrieved document: \\n\\n {context} \\n\\n\n",
    "    Here is the user question: {question} \\n\n",
    "    If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
    "    Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\",\n",
    "    input_variables=[\"context\", \"question\"],\n",
    ")\n",
    "\n",
    "# Chain\n",
    "chain = prompt | llm_with_tool | parser_tool\n",
    "\n",
    "print(convert_to_openai_tool(grade_tool_oai))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5f882689-623a-4b98-ba93-52b005e983f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:09.719573Z",
     "iopub.status.busy": "2024-11-27T11:21:09.719436Z",
     "iopub.status.idle": "2024-11-27T11:21:13.424477Z",
     "shell.execute_reply": "2024-11-27T11:21:13.424157Z",
     "shell.execute_reply.started": "2024-11-27T11:21:09.719559Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='', additional_kwargs={}, response_metadata={}, id='run-3b14c663-2f63-483e-b370-0c1363daca63-0', tool_calls=[{'name': 'grade', 'args': {'binary_score': 'yes'}, 'id': 'call_0cf57bc7cd434a7ca05a93ae8614acf9', 'type': 'tool_call'}])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(prompt | llm_with_tool).invoke({\n",
    "    'question': '报告的时间是什么时候', \n",
    "    'context': splitted_docs[0].page_content\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6871fc2-fe5d-40ad-a354-8a14dd2163fd",
   "metadata": {},
   "source": [
    "从结果看，触发了函数调用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bacaa93d-1cd6-43d2-86e9-0acc63bf3156",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:13.425144Z",
     "iopub.status.busy": "2024-11-27T11:21:13.424969Z",
     "iopub.status.idle": "2024-11-27T11:21:14.574242Z",
     "shell.execute_reply": "2024-11-27T11:21:14.573813Z",
     "shell.execute_reply.started": "2024-11-27T11:21:13.425132Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[grade(binary_score='yes')]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(prompt | llm_with_tool | parser_tool).invoke({\n",
    "    'question': '报告的时间是什么时候', \n",
    "    'context': splitted_docs[0].page_content\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4039a38f-e60f-4b39-a532-c273bf321e1b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:14.574847Z",
     "iopub.status.busy": "2024-11-27T11:21:14.574728Z",
     "iopub.status.idle": "2024-11-27T11:21:14.578217Z",
     "shell.execute_reply": "2024-11-27T11:21:14.577764Z",
     "shell.execute_reply.started": "2024-11-27T11:21:14.574835Z"
    }
   },
   "outputs": [],
   "source": [
    "def transform_query(state):\n",
    "    \"\"\"\n",
    "    Transform the query to produce a better question.\n",
    "\n",
    "    Args:\n",
    "        state (dict): The current state of the agent, including all keys.\n",
    "\n",
    "    Returns:\n",
    "        dict: New value saved to question.\n",
    "    \"\"\"\n",
    "\n",
    "    print(\"---TRANSFORM QUERY---\")\n",
    "    state_dict = state[\"keys\"]\n",
    "    question = state_dict[\"question\"]\n",
    "    documents = state_dict[\"documents\"]\n",
    "\n",
    "    # Create a prompt template with format instructions and the query\n",
    "    prompt = PromptTemplate(\n",
    "        template=\"\"\"You are generating questions that is well optimized for retrieval. \\n \n",
    "        Look at the input and try to reason about the underlying sematic intent / meaning. \\n \n",
    "        Please keep the generated question in the same language as the initial question. \\n\n",
    "        Here is the initial question:\n",
    "        \\n ------- \\n\n",
    "        {question} \n",
    "        \\n ------- \\n\n",
    "        Formulate an improved question: \"\"\",\n",
    "        input_variables=[\"question\"],\n",
    "    )\n",
    "\n",
    "    # Grader\n",
    "    # model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
    "    model = Ollama(\n",
    "        model='qwen2:7b-instruct',\n",
    "        base_url='http://localhost:11434'\n",
    "    )\n",
    "    \n",
    "    # Prompt\n",
    "    chain = prompt | model | StrOutputParser()\n",
    "    better_question = chain.invoke({\"question\": question})\n",
    "\n",
    "    return {\"keys\": {\"documents\": documents, \"question\": better_question}}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b36dd699-5a09-4d36-b593-071d92020d1f",
   "metadata": {},
   "source": [
    "使用Tavily进行搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9fad7fc5-f5e4-4370-8eca-bdb3efa2fcd9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:14.578891Z",
     "iopub.status.busy": "2024-11-27T11:21:14.578699Z",
     "iopub.status.idle": "2024-11-27T11:21:14.591895Z",
     "shell.execute_reply": "2024-11-27T11:21:14.591500Z",
     "shell.execute_reply.started": "2024-11-27T11:21:14.578879Z"
    }
   },
   "outputs": [],
   "source": [
    "def web_search(state):\n",
    "    \"\"\"\n",
    "    Web search using Tavily.\n",
    "\n",
    "    Args:\n",
    "        state (dict): The current state of the agent, including all keys.\n",
    "\n",
    "    Returns:\n",
    "        state (dict): Web results appended to documents.\n",
    "    \"\"\"\n",
    "\n",
    "    print(\"---WEB SEARCH---\")\n",
    "    \n",
    "    state_dict = state[\"keys\"]\n",
    "    question = state_dict[\"question\"]\n",
    "    documents = state_dict[\"documents\"]\n",
    "\n",
    "    print(f\"question: {question}\")\n",
    "    \n",
    "    tool = TavilySearchResults()\n",
    "    docs = tool.invoke({\"query\": question})\n",
    "    web_results = \"\\n\".join([d[\"content\"] for d in docs])\n",
    "    web_results = Document(page_content=web_results)\n",
    "    documents.append(web_results)\n",
    "\n",
    "    return {\"keys\": {\"documents\": documents, \"question\": question}}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf626f0c-bd70-499a-95ad-3ddf33441736",
   "metadata": {},
   "source": [
    "### Edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "181c8ead-ca10-4b1f-94d3-c4db1730b86d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:14.592593Z",
     "iopub.status.busy": "2024-11-27T11:21:14.592438Z",
     "iopub.status.idle": "2024-11-27T11:21:14.598030Z",
     "shell.execute_reply": "2024-11-27T11:21:14.597564Z",
     "shell.execute_reply.started": "2024-11-27T11:21:14.592581Z"
    }
   },
   "outputs": [],
   "source": [
    "def decide_to_generate(state):\n",
    "    \"\"\"\n",
    "    Determines whether to generate an answer, or re-generate a question.\n",
    "\n",
    "    Args:\n",
    "        state (dict): The current state of the agent, including all keys.\n",
    "\n",
    "    Returns:\n",
    "        dict: New key added to state, filtered_documents, that contains relevant documents.\n",
    "    \"\"\"\n",
    "\n",
    "    print(\"---DECIDE TO GENERATE---\")\n",
    "    state_dict = state[\"keys\"]\n",
    "    question = state_dict[\"question\"]\n",
    "    filtered_documents = state_dict[\"documents\"]\n",
    "    search = state_dict[\"run_web_search\"]\n",
    "\n",
    "    if search == \"Yes\":\n",
    "        # All documents have been filtered check_relevance\n",
    "        # We will re-generate a new query\n",
    "        print(\"---DECISION: TRANSFORM QUERY and RUN WEB SEARCH---\")\n",
    "        return \"transform_query\"\n",
    "    else:\n",
    "        # We have relevant documents, so generate answer\n",
    "        print(\"---DECISION: GENERATE---\")\n",
    "        return \"generate\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4ea8c76-6b64-4f82-a388-df660d32b54a",
   "metadata": {},
   "source": [
    "## Graph定义"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b447dd3b-ac1f-4c38-9388-35b114e045ea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:14.598586Z",
     "iopub.status.busy": "2024-11-27T11:21:14.598466Z",
     "iopub.status.idle": "2024-11-27T11:21:14.604811Z",
     "shell.execute_reply": "2024-11-27T11:21:14.604386Z",
     "shell.execute_reply.started": "2024-11-27T11:21:14.598574Z"
    }
   },
   "outputs": [],
   "source": [
    "import pprint\n",
    "\n",
    "from langgraph.graph import END, StateGraph\n",
    "\n",
    "workflow = StateGraph(GraphState)\n",
    "\n",
    "# Define the nodes\n",
    "workflow.add_node(\"retrieve\", retrieve)  # retrieve\n",
    "workflow.add_node(\"grade_documents\", grade_documents)  # grade documents\n",
    "workflow.add_node(\"generate\", generate)  # generatae\n",
    "workflow.add_node(\"transform_query\", transform_query)  # transform_query\n",
    "workflow.add_node(\"web_search\", web_search)  # web search\n",
    "\n",
    "# Build graph\n",
    "workflow.set_entry_point(\"retrieve\")\n",
    "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
    "workflow.add_conditional_edges(\n",
    "    \"grade_documents\",\n",
    "    decide_to_generate,\n",
    "    {\n",
    "        \"transform_query\": \"transform_query\",\n",
    "        \"generate\": \"generate\",\n",
    "    },\n",
    ")\n",
    "workflow.add_edge(\"transform_query\", \"web_search\")\n",
    "workflow.add_edge(\"web_search\", \"generate\")\n",
    "workflow.add_edge(\"generate\", END)\n",
    "\n",
    "# Compile\n",
    "app = workflow.compile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b4a0ef4d-c43b-48b4-ba75-f1f1da8abd9c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T15:04:04.290604Z",
     "iopub.status.busy": "2024-11-27T15:04:04.289835Z",
     "iopub.status.idle": "2024-11-27T15:04:08.545454Z",
     "shell.execute_reply": "2024-11-27T15:04:08.544528Z",
     "shell.execute_reply.started": "2024-11-27T15:04:04.290533Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:547: UserWarning: <built-in function any> is not a Python type (it may be an instance of an object), Pydantic will allow any object with no validation since we cannot even enforce that the input is an instance of the given type. To get rid of this error wrap the type with `pydantic.SkipValidation`.\n",
      "  warn(\n",
      "/opt/anaconda3/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:547: UserWarning: <built-in function any> is not a Python type (it may be an instance of an object), Pydantic will allow any object with no validation since we cannot even enforce that the input is an instance of the given type. To get rid of this error wrap the type with `pydantic.SkipValidation`.\n",
      "  warn(\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAJ2AOADASIAAhEBAxEB/8QAHQABAAMBAQEBAQEAAAAAAAAAAAUGBwQIAwECCf/EAF0QAAEDBAADBAMJCwYLBQUJAAEAAgMEBQYRBxIhExQxQRVWlAgWFyJRVdHS0yMyNkJUYXF1lbPUNXSBkpOyJTdDRXJzkaGxtPAYNFLBxAkkRFe1JjNTYoKWoqPC/8QAGwEBAQEBAQEBAQAAAAAAAAAAAAECBAMFBwb/xAA3EQEAAQICBwUHAwQDAQAAAAAAAQIRAxIEFCExUVKRQXGhwdETM2FikrHSQoHwBRUiIzJD4fH/2gAMAwEAAhEDEQA/AP8AVNERAREQEREBERAREQERc1yuNPaaGasqn9nBE3mcQ0uJ+QADq4k6AABJJAGyVYiZm0DpXDVX220MhZU3Clp3j8WWdrT/ALCVCtsVZlLRUXx81LSP2Y7PBLyNa0jp272nb3/K1p5BvXx9c576bC8fo2BkFitsTQANMpIx4eHkvfLh07KpvPw9f53rsfT31WT54oPamfSnvqsnzxQe1M+lfvvWsvzRQezM+hPetZfmig9mZ9Cf6fj4LsfnvqsnzxQe1M+lPfVZPnig9qZ9K/fetZfmig9mZ9Ce9ay/NFB7Mz6E/wBPx8DY/PfVZPnig9qZ9Ke+qyfPFB7Uz6V++9ay/NFB7Mz6E961l+aKD2Zn0J/p+PgbH6zJ7NI4NbdqFzj5CpYT/wAVIse2Roc1wc0jYIOwVGOxSyPaWus9vc09CDSsIP8AuUccBt9C4zWIuxyq3zbt7Q2F5+R8P3jgfM6DvkcD1S2DO6Zj9v59pTYsyKIsd6lrZJ6KuhFLdKbXaxtO2SNPhLGfNh0fHqCCD4bMuvGqmaJtKCIiyCIiAiIgIiICIiAiIgIiICrF61dc0s1sfp1NSwyXOVh38aRrmsh/SAXPd182NP6LOqxUjufEehldsMrrbLA12unPHI14bv5S17yP9A/m30YP/KZ7bT9lhZ0X45wY0ucQ1oGyT4BUD/tC8K//AJl4f+3qX7Rc6NAWa2Hjpb8ozK7WC0Y3kdxhtlZUW6ovUNHH3AVcDC6SHnMgcHAjlDi0NLiBzdV1f9oXhX/8y8P/AG9S/aLNqXDcqquP1BkmNYm/EbPLXSy3u9097hmoMgouxc2F5pGHfbl3ZuEha0tAO3v2gl+CPH+95/w3u2RXzCr5BNQz1vL3GlhkbVtjq5YmwwRsne90rWsa1/MGguDi0kaKmaD3R9jnx/NbhcbDkOP12I243W42W60kcVYabs5HtkiAkdG8O7J4Gn9C3R0s1peH/FC08G8z4cW2xS0M8dfWVtuyClu8MLblTzXLvL6dmndrBI+GWWPmc0BpH33XagY+BeShnFY2LhlBhttybBJbRb7bFcaWSR1c3tgBOWv5Q+Ttxpwc5uo9ueCdIL3nvuoLrQY9iV4xzA8iloL1fLfRRz11LTs75TTkk9gx1S1zXvAAaZA0ddnQ0VvVnr5LraaOsloam2S1ELZXUVZydtASNlj+Rzm8w8DyuI2OhKyTixgWSXbhbg7LHbY7jfcYulpuzrS6oZCakUxHaQtkceRrtF2iTy7HirMzjliFohhp8vyGw4TkHIH1Niu98pG1NLvq0P1Jrq3TtgkaIQaCioB90HwtaATxKxAAjY3fqXr/AP2K1Y3ldkzK2i44/eKC+W8vMYq7bVMqIuYeLedhI2N+G0EbmJFsrLJemANkp6yOjld126Goe2It/R2joX//AKFZ1Wc+Z3u222gaCZay6UjGgDfSOZsz/wBH3OJ/VWZdFe3Domd+3p/9us7hERc6CIiAiIgIiICIiAiIgIiICisispvVCxsUgp66mkFRR1DmlwimaCASAQS0glrhsba5w2N7Uqi1TVNE5oNyIs+QxXKV9HUM7jdoh92oZD10Px4yQO0jPk8D8xDXBzR3ejaT8lg/sx9C+F4sFvv8LI7hSR1IjPNG89Hxu1rbHjTmnXTbSCoc4LydKfIb9TRjoGCt7XQ/TI1zj+kna9rYVW29vGPX+b12J/0bRj/4WD+zH0LoAAAAGgPJVf3kT+tN+/t4vsk95E/rTfv7eL7JPZ4fP4Sto4rSiq3vIn9ab9/bxfZKqcL7bdMuxEXK4ZTeRUmvr6b7hNEG8kNZNCz/ACZ68kbd/n34eCezw+fwktHFqi+ElFTzPL5IInuPi5zASq77yJ/Wm/f28X2Se8if1pv39vF9kns8Pn8JLRxWD0bR/ksH9mPoXyrrhb8eoXVFVNBQ0rSBzPIYC4+AHyk+AA6k+ChRhE3UOye/OB8u8Rj/AHiMFddrwu12urbWdnNW1zfvauvqH1Erfl5S8nk/Q3QTLhRvqv3R6/8AqbHytNHUXm7tvldA6mbFG6KgpZAQ+NjuUukkHk93KAB4tb0Oi5wFiRF5V155J2iIiwgiIgIiICIiAiIgIiICIiAiIgIiICIiAs+4EEHh23lJI9L3fx/WVT+c/wDXyeC0FZ9wI38Hbd8v8r3f70DX8pVPyf8AXy9UGgoiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICz3gMNcOhpwd/he79WjX+cqlaEs84C6+DkaJI9MXfxGv851KDQ0REBERAREQEREBERAREQEREBERAREQEREBEUdfb3BYLeaqZr5XF7YooIht80jjprG76bJPiSABskgAkappmqYpjeJFFSn37Lnu5mW6zRNPgx9ZK4j+kRDf+z6V/PpzMPyGx+1TfZrq1WvjHWFsu6KkenMw/IbH7VN9mnpzMPyGx+1TfZpqtfGOsFkf7oHivX8EuGVwzGixt2URW57HVdGyr7s6OAnRlB5H83KS3Y14EnfTrh/uGfdPz8a33jG4MOltNBa+9XGW7GtErDJUVj5WQcgibo6kk+Nvr2fh16breJclv8Aaa22XC1WGqoK2B9NUQSVM3LJG9pa5p+5+BBIWb+534K3H3OOG1VhscForTV1j6uorameQSyE9GNOo/BrQAPz7PTek1WvjHWCz0UipHpzMPyGx+1TfZp6czD8hsftU32aarXxjrBZd0VI9OZh+Q2P2qb7NPTmYfkNj9qm+zTVa+MdYLLuiruP5PUVtabddKOOguXZmWMQSmWGdgIDixxa07BI20ga2NbHVWJc9dFWHOWotYREXmgiIgIiICIiAiIgIiICIiAqfxBP/vmJjyN36g/zWoP/AJK4Kn8Qv++4j+uP/SVK6tG97H7/AGlYdqIi6UERcN0vlvshoxcK2CjNZUNpKYTyBpmmcCWxs3984hrjoddAnyQdyIom2ZVa7ze7zaKOpMtws74o66Hsnt7J0kYkYOYgB22kH4pOvA6KglkRFQREQQ9R0zzGNeJbVj+js2/QFe1Q6n8PMY/0av8AdhXxeOk/o7vOVnsERFxIIiICIiAiIgIiICIiAiIgKn8Qv++4j+uP/SVKuCp/EL/vuI/rj/0lSurRvex3T9pWHasl91PcrrauDVdNZLtVWS5PuVrgir6KQsli7Svp2O0R4gtcQQehBIOwStaUPleI2nN7M61Xuk77QOmhqDD2j4/ukUrZYzthB6PY063o60djYXRO2EeeuI1vya38Tce4a41crzWUFTa6q+1ElfllTQ1VZM2WOLs21YimkDWAl/Yxhg+PvoG6MLneAZU/HeF9uz671hq2Z/HTUUttvs75mUUsEpYJahjIS+ZjmOaJeUOAPQ7c7fo3PeF2L8TYKKPI7WK51DIZaWojnkp56dxGnGOWJzXt2NAgOG9DfguSs4M4bX4JT4dPZGPx6nlE8NMJ5Q+OUPMnatlDu0D+dznc4dzbJ69SsTSM3vmM1mQ+6JpMSdlOSUON0WFw1DqShu88MlRMKt8bZHytdzl/KPjO2HOIHMSNg0HO83yfBLrxWtNkvVxl7zl1ktVNPcrpI4W2Ktp4pJuzlkEggaXPc0EMIZzghp5QF6VsXDfHsau9NdbfQyR3GntrbRHUS1U0zu6tkMgYed55jzknmO3fn0vlceFWJ3cZS2vssFbHk5iN3jqXOkZVGONscZLSSGlrWN1ygdWg+PVMsjEGcMOL1PY8tpYLxV2umktQqLXE7Kqi7VbbnDK2SPU0kETmwyta5j2FzmnfQdSq3lPHXIMpwjL+KOMV9Tb7PS0FrsFvhkncynhqqmWF1bUyNILeeETxwh7mO5XRydCNg+kcC4VYxwyFZ73bfLSPrBGJ5aisnqpHtZzcjeeZ73creZ2gDobK7afh/jlLi9bjkdmpPQVa6ofUW90fNFKZ5HSS7ad9HOe468BvQ0AEyyMw4Q4RxFxbN+3u9VI3F5aCRlRSV+UzXuV9TzsMcsbpaaIxDl7QOaHFp5m6aNLb1S8B4OYjwxnqJ8ctklFNPE2B8k1bPUuEbTtsbTK93K0E9Gt0PzK6LURaBDVP4eYx/o1f7sK+Kh1P4eYx/o1f7sK+Lz0n9Hd5ys9giIuJBERAREQEREBERAREQEREBU/iF/33Ef1x/wCkqVcFC5VYpb5QQ92lbDXUkzammdJvk5wCOV2uvKWuc068N7660ujAqijEiav5eLLDmRQz7jkER5XYjXSuHi6CrpSz+gulaf8AaAv59LX/ANTbn7VR/brvyfNH1R6lk2ihPS1/9Tbn7VR/bp6Wv/qbc/aqP7dPZ/NH1U+q2TaKE9LX/wBTbn7VR/bqNx/N7hlNt7/bMUulRSdtNT9oZ6Vnx4pXxSDTpgej2OG/PWxsJ7P5o+qn1LLaihPS1/8AU25+1Uf26elr/wCptz9qo/t09n80fVT6lk2ihPS1/wDU25+1Uf26elb/AOptz9qo/t09n80fVT6pZ/VT+HmMf6NX+7Cviqdgs1wrbxDeLrStt5ponxU1GJRI8c/LzPkLfi7+LoNbvQJJJ5tNti49JqiZpiJ3R5zPmSIiLkQREQEREBERAREQEREBERAREQEREBERAVA4GDXDxvTX+FrsfDX+can8w/68z4q/rPuBDeTh20aLf8L3c6cNf5yqUGgoiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICzzgMQeHTdHY9L3fy1/nKpWhrP+BQcOHbeYvJ9LXb78aOvSVTr+jXh+bSDQEREBERAREQEREBERAREQEREBERARFyXO7UNlo31dwrKegpWffT1MrY2D9LnEBWImZtA60VWPFPDgdHKLSD/ADxn0p8KmHetFp9sj+le+rY3JPSWss8FpRVb4VMO9aLT7ZH9KfCph3rRafbI/pTVsbknpJlngtKKrfCph3rRafbI/pT4VMO9aLT7ZH9Katjck9JMs8FpRVb4VMO9aLT7ZH9KfCph3rRafbI/pTVsbknpJlngk8ly6xYXb2V2QXq32KhfIIW1Nzqo6eMvIJDQ55A2Q0nXj0PyLNfc38QMVyLEnWqz5JaLnc47hdat9DRV0UszYXXGciQsa8uDD2jCHeB52/KFzceYsB428KMgxGsyezNfWwE0s76uP7hUN6xP8emnAb/MSPNecf8A2dPDqycIbRkOVZZdbfa8kuMjrdT0tTUsa+KljeC52t+Ej2gj8zAfApq2NyT0kyzwe90VW+FTDvWi0+2R/SnwqYd60Wn2yP6U1bG5J6SZZ4LSiq3wqYd60Wn2yP6U+FTDvWi0+2R/SmrY3JPSTLPBaUVW+FTDvWi0+2R/SnwqYd60Wn2yP6U1bG5J6SZZ4LSiq3wqYd60Wn2yP6U+FTDvWi0+2R/SmrY3JPSTLPBaUURZsuseRSuitd4obhK1vO6OmqGPcG71zaB3rfTal141U1UTaqLSyIiLIIiICIiAqNVOFzz64ioAlbbaeAUzXdRG6TnL3geHMQGjetgDW+pV5VEh/D/Jf9VSf3Xrt0XfXPw84ajtTCIi9mRERAREQEREBERAREQEREBERAREQV3PuWkxa43VoDKy1QSV9NO0fHjkjaXDR6dCAWkb05rnNOwSFoqzjiV/i5yr9VVf7ly0dY0j3VE/GfJrsERF89kREQEREBUSH8P8l/1VJ/der2qJD+H+S/6qk/uvXbo36+7zhqN0phZRxE41XPF8nudjxzEnZVV2e1NvN1c64Mo2wQOc8MZHtju0ld2UhDfijQHxuq1deSfdVPpbVxVpqt98tNmbW2AUVXS1d1r7ZJcoTNITE6SCnlZKNbDWtLJG8z97Dm63VNoZaNR+6Fvl9r8NtdgwyG83PI8WZkzSbt3enp2l0YLHvdCXcv3QAODSeYgcoBLh/WZ+6OqOHmfUVjv+P22lttVW09EyqiyKnfXHtnNY2YUXKHmIPcAXb5gNnl0vvwrxmuyDJ8N4hR2eLFrP7yxaG2CYuFRRvdPFIxgaWAdm1kegTp33vxR11S8m9zpm9ZT5ZbbZLij6W65GMkju9f2/f5nNqGTx0smmEMa0sDBIHP0waEY3sZvVYTfGXjXkE+P8TbZhWNVFxpsctlTBcsiZdG0Zoqo0xk1Tt5S6R8TXMe7TmaOgCSuWu91BS4bRY5j8fom5X2Ow0VfcJ8hySC1MHaxAta18wc6aV3K5xAGgC0ucOYLqyzg1xApPhNtOI1mOS43nLaiplbeXzx1FDVT0wglLOzY5sjHcjXDfKWkno7z+tNwZzXBsjhv2Iy43cai4WSgtl4t9+MzYhNSxlkc8EkbHO1yuILC0b0DseT/K41LhfxDt3FbA7RlVqZJHRXGNzmxylpdG5r3Me0lpLTp7HDYJB1sdCobiLxTrMVyOy4vjuPuynKrrFLVMoTVtpIaeliLQ+eaUtdyt5nsaAGkuJ15L61HE624VFR2rII7i+8xU0bqp1jxy41FIZC0FxjdFDI0N3vTS4kefVVG+Wu85vmll4kcN5qdlxoKWosVdb8roK23R1VM90c2288Qka5r2tIdyOa7ZGxorV9gqXDbjdfrTj1xp620Vt9zO95vdrbbbDPcg4U4icZHsNQ4ODIYWNd1a0+XK3qrZL7pQ2623GiuWK1UGd0l4p7EzGaerZN3ipnj7WB0dQQ1vZOjDnl7mjlDHbbsDcDZ+AebWilpL/HX2D38W/K7nkFPHzTG3zwVrXMlge7l52HldsOAdotH32+n7V+55y27z3DMqq82il4kS36kvdMyBkslshbTU7qeOlcSBI5ro5JOaQAHmdsN6dcxmsLrW8Wslxqix92T4XDaKy75DS2OOGnvDaqMMma49uHiJpPKWkcha3fy68eLiB7oX3iV2d0/vdkuXvWis0uoqsMfV9/qHQ8rQ5umlnLsbcQ7ets8V/GV4LxFz/FqCS7vxigySy3yjvVsp6GWokpH9j99HNI5gf8YOf8ZrPi9OhVVyHgRnuZu4jVt2qsdpq7JxYRS09HPO6KmbQ1TppWve6IF22n4rg0bJ0WtA2bMz2CcqOO+cU+TXzGzwzp33u1W5l4fGzImGCSkeXtHLJ2G+1Lo3jk5eX4p+OBonmyX3W9ioKbFhaYrZLW32zxX0R5BfILRDT00nRgdJIHc0hcHDkY0/eEkga3dpuHNyk4uZJlInpfR9yxqns0MRe7tWzRzVD3OcOXQZqZuiCTsHp4bzbGOAOa8OYsNuuO1GOXC+UGM02OXi3Xd0wo5xC4vjmhlbGXtc1znjRZpzT5EbT/ITFq90+/L4cLjxXFhebhkgucQhlusUUNJPRPjbIHTNa9r4zzuIkj5tgN008x5YXOOPWW3TD8Ur8ZssNtu5zWLHLxbqq4NAbNHMWOpxKIX80cmh91DQQ0j4p2QtBdw8yK5Z5w2yW5S2hktgorlDc4qASRsfLUthDOwa4HbR2Tt8zgfDx3oVO5cBclfi97joa+1Mvgzt2Y2sTukNM9oma9kM5DeZpLQ4EtDtHWiVP8hO1nGrII+I9DhNJhtNVXttsprlcmPvbIWwtle5jhTc8QNSGFji52meXmdLXlgfFjhVxD4tUdqpKunwy2yw9hM29U8lS64WmobNzPfSP7Mc22BoAJj673sdFvi3FxW+JX+LnKv1VV/uXLR1nHEr/FzlX6qq/wBy5aOppHuaO+r7UtdgiIvnsiIiAiIgKiQ/h/kv+qpP7r1e1RqwNtWeXB9S4RMuUEBpnv6NkeznD2A+HMByu1vZBJA+Kddui764+HnDUdqWREXsyIiICIiAiIgIiICIiAiIgIiICIiCt8Sv8XOVfqqr/cuWjrOs9LKzGLhaGEPrrrBJQ01O0/HkfIwt6Dr0AJcTrQa1xOgCVoqxpHuqI+M+TXYIiL57IiIgIiIC5bjbKO8Uj6WvpIK6lf8AfQVMbZGO/S0ggrqRWJmJvAq54WYYTs4nZCf1fF9VfnwWYZ6pWT9nxfVVpRe+sY3PPWWs08VW+CzDPVKyfs+L6qfBZhnqlZP2fF9VWlcV2u9LZKM1VZI6OLmawBkbpHvcToNaxoLnEnwABKaxjc89ZM08VZunD7BbPbaquqcTtHYU8bpXiC0smkIaNkMjYwue7p0a0FxOgASdLioOEWNV9Way4YpZadkUpfR0sNGxpawxhp7fXxZHbLzrq0bb0Lm8ytFBaZ6i4NuV0EZrYHTxUrKeV5ijhe8cpLToOkLWMJcRtvM9rTouLphNYxueesmaeKrfBZhnqlZP2fF9VPgswz1Ssn7Pi+qrSiaxjc89ZM08VW+CzDPVKyfs+L6qfBZhnqlZP2fF9VWlE1jG556yZp4qt8FmGeqVk/Z8X1U+CzDPVKyfs+L6qtKJrGNzz1kzTxUm6cG8Nr2RPixmz09VTl0lPIKJojEhjcwGRjeUStHOTyO6bDT0c1pHwsuBYhWulo6zDbHBdKWOE1TI7awwlz2b5onuYOdnMHtB0Dth2Ar6uG62mK6MgLy9k9PIJoJGSPZyvHhzcjmlzflYTpw6FNYxueesmaeKE+CzDPVKyfs+L6qfBZhnqlZP2fF9VSVhvZrnSW+tlo2X6jijfW0dJK54jD+bkeOZrXFjuR+na1tj27JaVLprGNzz1kzTxVb4LMM9UrJ+z4vqp8FmGeqVk/Z8X1VaUTWMbnnrJmniirNidkx17n2qz0Fte5vI51JTMiJbveiWgdN9dKVRF41VVVzeqbyzvERFkEREBERAREQEREHwra6mtlLJU1lRFS00Y2+ad4Yxo8OpPQKMs1LV1lQ27XGGegrHMkgbb21plhji7QlrnNaAwyuaGF333IeZrXlu3P46qppshy70S2rpaiG0xR1ddb5aQyO7V7+aleJHfFbymGR+ht2+zdtoA57MgIiICIiAiIgIiICIiCGyaKphpY7nSPrXy27nqXUNC2N769gjdun08gbcSC08zSHtbt3LzB0pTVDauminY17WSMDw2VjmOAI31a4AtP5iAQvqoDHaJ9luN0tsVFNDbu077T1MlV2wkfM97pWNafjMDHdeXq3UgDdAcrQn0REBERAREQEREBERAREQERRmUPu0WNXZ9gjppb62kmNvZW77B1RyHshJog8nPy70Qdb6hBwYzdfSF+y6D0wLl3G5R0/dBS9l6P3R00vYl/8Ald9r2vP5dsGfiFWJeFfcle6y4zcc+NVyxu8WrH6SzUjn1V2cygmZJQtYxsQgiJm8XSN3905yC6T8VrWt91ICIiAiIgIiICIiAiIgKu3639nlOOXaC1xVlTG+a3zVrqjsn0lLKzncWt8JOaaCmaW+I3zD70g2JVzPqHvuO87bdT3Sakq6Wsigqp+wYHxTseH8/kW8vMN9CRo9CUFjREQEREHzqKiOkp5Z5XcsUbS9zvkAGyVQ4Z79k1PDcRfamxwVDBLDR0UEDixhG287pY3ku146AA8Outm2ZV+DF4/mc39wqvYz+Dlq/mkX9wL6GjxEUTXaJm9tsX+7W6Lub0PffXS8ez0P8Mnoe++ul49nof4ZTaL3z/LH0x6F0J6HvvrpePZ6H+GT0PffXS8ez0P8MptEz/LH0x6F0J6HvvrpePZ6H+GT0PffXS8ez0P8MptEz/LH0x6F0J6HvvrpePZ6H+GT0PffXS8ez0P8MptEz/LH0x6F2e4pwbpcHvmR3ix3652+5ZFUisulRHDRk1MoB04gwEDxcdN0NuJ1skqz+h7766Xj2eh/hlNomf5Y+mPQuhPQ999dLx7PQ/wyeh7766Xj2eh/hlNomf5Y+mPQuhPQ999dLx7PQ/wyeh7766Xj2eh/hlNomf5Y+mPQuhPQ999dLx7PQ/wyeh7766Xj2eh/hlNomf5Y+mPQuhPQ999dLx7PQ/wykLDeLhb7zBaLpV+km1Ub5KasdG2OTbNc0cgaA09DsOAHgQR02etQtZ+HWK/pqv3KbMSJpmI3TOyIjdF+yCJuvqIi+QyKv8QaBt0wPIqR1BT3US2+dooaubsYag9m7Ub5PxGk6Bd5A78lYFFZVSitxe8U7qaCsbNRzRmmqn8kUoLCOR7vJp8CfIEoJKF/axMeQAXNB0Dsf7fNf2uGxAiyW8OiigcKePcUD+eNnxR0a7zA8j5ruQEREEXlX4MXj+Zzf3Cq9jP4OWr+aRf3ArDlX4MXj+Zzf3Cq9jP4OWr+aRf3Avo4PuZ7/JrsSShsRzC1ZzZvStmqDVUBqJ6YSmNzNvhlfFINOAPR7HDfnralainjqoJIJmNlhkaWPY4bDmkaIP8AQvDtJjGJWL3G3EttporZb8l7e5UleKZrG1TWRXR7Y2SAfG0xhjAB8AWqTNmXuZF5w4x4LiGNU+IYHbcWsb35JcZqp1Zf5pRSCWCn+PNU8j2uqZXNIAa53xjtxILdrMMYgiyLhrg+PVlxhvNpouLE9oifQVEggdSCGqIijd2jn9kWvIDS93xCASR4yarTYe3lV+IXEK38ObRSVlbTVtwnrayOgorfbohJUVVQ8EtjYCWt3prnEucAA0klebb1w8oZ6vi3gdquttxXH6G92SuobXcZnQ2yaR9O2SSjeA4FsUzmdWs8zsA+CjLnZcH4l4nwhi95Vvt8FDns1gqrf2orKduoqt08cMp/+8gdIxjh0A+KBocqTVI9h2qtfcrZSVclJUW+SeJsrqSqDRLCSN8j+Vzm8w8DokbHiV1LyHmtrxC+3jjlcM+rYqa+46AywmesdDJbaQUTH08tIOYac6YvO29XOHL18FKYLh1PxQ4zWmoz62i53BvDeyVVXQ1oJi726aoL3vj8C9rubWx8UuOkzdg9UIvEOTYHZK/htnuTS0j/AHw03Eyalp7myeRlRTwyXiON8cT2uBja5ssmw3QJcT49Vt3CrHrdgfH/AIgY3YKVlrsLrPa7iLdBsQR1D31LHyMZ4NLmxs5teJaCeqsVXG4Isu909eLvYOAmZV9jqKikr4aME1NLvtYITIwTyM11DmxGRwI8Nb8lilzxDA8e4k4tScOTRzU9fiF+klFvrDUmpd2VOIpX/Gdt7uZ/xz1d16nXRNVh68ReTbXkturcL9ypR0FfSVdyb3V7aVkzS/7naJmP2AdgNeQ0/ITpVHgzhNXn1qxjK6jOsWs+cTXVslbUyUlQ2+GqjmJmo5HOrQ0gta5nZdly8h6NHQqZh7gReYuGHBHFeJVJxelvdujrLjWZXfKCGtnHO+ijfuM9jvow/HcSRokkbPQa+nAe/wBz4vcQrTWX2N4rOHNoks1f2gIDr1JI6GoeD56hpg4H5Kn9KuYemVC1n4dYr+mq/cqaULWfh1iv6ar9yvej9XdV9pWF9REXyEFH5BD3iw3KLu8VX2lNK3u87uWOXbCOVx8gfAn5CpBcF/h7xYrjF3dlZz00je7yv5Gy7aRyl3kD4b8toP4xmIwY3aojTQ0ZZSRNNNTv544tMHxGu82jwB8wFJKMxen7pjVpg7pHQdnSQs7rDJ2jIdMA5Gu/GA8AfPSk0BERBF5V+DF4/mc39wqvYz+Dlq/mkX9wKxZQ0uxm7NA2TSTAAf6BVdxkg43aSCCDSRdQd/iBfRwfcz3+TXYklW6nhriNZWXOrqMVsk9VdGdlXzyW6Fz6tmweWVxbuQba06dvq0fIrIirKKyPFLJmFA2hv1noL3RNeJBTXGlZURh48HcrwRsbPVfGLBsbgfG+LH7VG6OpZWscyijBbUMZ2bJR8Xo9rPih3iG9AdKbRBB3TBMavjbi25Y9arg24mN1aKqiilFUYxyxmXmaeflHQc29DwX0gw6wUtDbKKGx22KjtcoqKCnjpI2x0kgDgHxNA0x2nvG26OnO+UqYRLCCvWB41kl0o7ld8dtV1uNHru1ZW0UU00Gjscj3NJb169CpFlmt8d3luraGmbdJYW00laIWiZ8TXFzYy/XMWgucQ3egXE+a7EQQ78OsElDU0T7HbXUVTVd/npnUkZjlqe0EnbObrTpOcB/OevMAd7XHkmGel3zVdquMuL3qcRxz3i20lK+qmiZzcsTnTRSAsBe4ga6EnRGzuyIlhTcfwe+Wu4tnuee3rI6Pkcx9vuFHb2QybGuphpmP6fmdr5dqIpuBOPWXiVj2V4/QWzHI7XS11PNQW22xwCrdUdj90c5nLot7E+LTvn8RrrpKJaBW6DhniFquPf6LFLJR1/eO996gt0LJe25XN7XmDd8/K945vHTiPMr6s4f4vHkjshbjdobf3ffXUUEQqj011l5ebw/Op9EsIr0BDbrbc4bHFSWWrrXTVBqIaVpb3l46zvYOXtHc2idnbtdSoThhw6j4cWWup33CS8XW510tzudzlhZE6qqZNcz+RnRjQGtaGjwDR1J2VcESwKFrPw6xX9NV+5U0oarBOdYtryNUT+jstf8AmP8AavWj9XdV9pWF8REXyEFwX+HvFiuUXd2VfaU0je7yP5Gy7aRyl3kD4b8trvXBf4e8WK5Rd3ZV9pTSN7vI/kbLtpHKXeQPhvy2g+eL0/dMZtEHdI6DsqOFndYZO0ZDpgHI1/4wHgD562pNRmL0/dMZtEHdI6DsqOFndYZO0ZDpgHI1/wCMB4A+etqTQEREH45oe0tcA5pGiD4FUt+HXq1fcLJdaJlub0ip7hSPlfC3/wALZGyN20eABGwPMq6ovbDxasL/AI+qxNlI9AZh852P2Cb7ZPQGYfOdj9gm+2V3Re2tYnCOkLdSPQGYfOdj9gm+2T0BmHznY/YJvtld0TWsThHSC6kegMw+c7H7BN9snoDMPnOx+wTfbK7omtYnCOkF1I9AZh852P2Cb7ZPQGYfOdj9gm+2V3RNaxOEdILspxuty/Ir3lduFTZac2G5R28yOo5iJy6jpqnnA7XoB3nl11+8J89Cf9AZh852P2Cb7ZcvDSQPzfiwA3RZkkAJ6df8D2076D8/nvw+TQGgprWJwjpBdSPQGYfOdj9gm+2T0BmHznY/YJvtld0TWsThHSC6kegMw+c7H7BN9snoDMPnOx+wTfbK7omtYnCOkF1I9AZh852P2Cb7ZPQGYfOdj9gm+2V3RNaxOEdILqR6AzD5zsfsE32ylbBi89DWm43StZcLjyGKMwwmGGBhIJDGFzjs6G3EknQ1odFYkWKtIxKoy7I7oiEuIiLmQXBf4e8WK5Rd3ZV9pTSN7vI/kbLtpHKXeQPhvy2u9cF/h7xYrlF3dlX2lNI3u8j+Rsu2kcpd5A+G/LaD54vT90xm0Qd0joOyo4Wd1hk7RkOmAcjX/jAeAPnrak1GYvT90xm0Qd0joOyo4Wd1hk7RkOmAcjX/AIwHgD562pNAREQEREBERAREQEREBERBn/DVxdm/FcGQycuSQAN2Tyf4Htx11HTx30+X5drQFn3DSUyZvxZaRoMySBo6k7/wPbT5np4+WloKAiIgIiICIiAiIgIiIC4L/D3ixXKLu7KvtKaRvd5H8jZdtI5S7yB8N+W13rgv0PebFcYjTsqxJTSN7vI/kbLtpHKXeQPhvy2g+eL0/dMZtEHdI6DsqOFndYZO0ZDpgHI1/wCMB4A+etqTUZjFOKTGrTAKWOhEVJCzusUnaMh0wDka78YDwB89KTQEREBERAREQEREBERARFH5DfqHFbBc71dJjTW2200tZVTCN0hjijYXvdytBc7TWk6aCT5AoKdw0LTnHFnQaD75KffLvZPoe2+O/Pw8Omtee1oK8ycEfdX8L8x4pZjZ7PlMtyuOR32KotNNHbK37rE22UcTjswgRgPgm3zEa5S7wO16bQEREBERAREQEREBERAXBf4e8WK5Rd3ZV9pTSN7vI/kbLtpHKXeQPhvy2u9cF/h7xYrlF3dlX2lNI3u8j+Rsu2kcpd5A+G/LaD54vT90xm0Qd0joOyo4Wd1hk7RkOmAcjX/jAeAPnrak1GYvT90xm0Qd0joOyo4Wd1hk7RkOmAcjX/jAeAPnrak0BERAREQFXrxmlLa66SihpKy6VcQBmioYw7stjYDnOc1oJHXl3vRB1ojdhWe4S4y2epmd1kluNc57vMnvUo/4AD8wAC68DDpqia6ttreN/RY4pL4RJPVa/f1Kf7ZPhEk9Vr9/Up/tl2oujLhcnjPqt44OL4RJPVa/f1Kf7ZPhEk9Vr9/Up/tl2omXC5PGfUvHBxfCJJ6rX7+pT/bL5VWdNrqaamqMRvc9PMwxyRSRU7mvaRoggzdQQpJEy4XJ4z6l44PJfuX/AHN0XAHivmeVT47dK2nqJHU+PsY2B0lLSvdzP7Tcg0/71nxSegd/4l6n+EST1Wv39Sn+2XaiZcLk8Z9S8cHF8Iknqtfv6lP9snwiSeq1+/qU/wBsu1Ey4XJ4z6l44OL4RJPVa/f1Kf7ZPhEk9Vr9/Up/tl2omXC5PGfUvHBxDiI/fXF760fKY4D/AMJlYLLfKS/0jqikc74jzHLFKwskieNEte09QdEH84II2CCYxRuPOMfEO7Mb0bJbKV7h8rhLOAf06Ot/o+QLNeFh1UVTTFpg3rqiIvnMiIiAuC/w94sVyi7uyr7Smkb3eR/I2XbSOUu8gfDfltd64L/D3ixXKLu7KvtKaRvd5H8jZdtI5S7yB8N+W0Hzxen7pjNog7pHQdlRws7rDJ2jIdMA5Gv/ABgPAHz1tSajMXp+6YzaIO6R0HZUcLO6wydoyHTAORr/AMYDwB89bUmgIiICIiAs8wX+QZP5/Xf83KtDWeYL/IMn8/rv+blXfo/u6u+PNrsWBEXnS/8AupazDbfV094tcE99t+WT2mvp6Rjw2G1xATuruUuJAbSvicSToud8nQamYjey9FovPWYe6XuNhdkktDQ0lXSe+OHFrFIKaonMlS2Ay1k0rYQ98kcZDmhsTOYmMjfXbaJxP4r5PnHBvObTd7Wx09DNZqijvMVor7ZR1RfcoAYjHVsD2vYWtJLS8FrxrRBCk1wPYCLJ8P4hZhbuKnvGzmms0lRX22S62u52JsscUjI5GMlhkjlc4h7e0Y4OB0QfAHor9m15r8dw2+3W12195uVDQzVNNboyQ6plYwubGNbO3EAdAT1Wr3E0i840XulbtbuED8sr6vG8juFbXUlst9LYIatvd6uY6dDVQntJQ6PqS1redwaQGgkLuwzjtlVxdk9HeKa2wigs77pS5HVWe5We2Rva7lMNQKtnMNczXczHHbeboCFnNA9AIvMVj91BkUtj4isq2Wa6XKwYvJklsuFDbq6jpKhrRIOR8VSQ9wDmNPOx3K4O6EEFWmu4q8QbPj+KtqaLHqrKc0qYILPb4WTxwUAMD5531Mhe4yhjG+DGs2enXxTNA3RF5w41jP4bTwzbd345WZKM7pBRSULJ4KNzTSVPK6VrnPeCCXbDXHYA6jfTR+E2eZBkF9zHGcrgtrb9jdVBG+qtDZGU1TDPCJYntZI5zmOHxmkFx6joequbbYaQoyw/4x7l+qab99MpNRlh/wAY9y/VNN++mXp/1193nDUdq7IiL5TIiIgLgv8AD3ixXKLu7KvtKaRvd5H8jZdtI5S7yB8N+W13rgv8PeLFcou7sq+0ppG93kfyNl20jlLvIHw35bQfPF6fumM2iDukdB2VHCzusMnaMh0wDka/8YDwB89bUmozF6fumM2iDukdB2VHCzusMnaMh0wDka/8YDwB89bUmgIiICIiAs8wX+QZP5/Xf83KtDWeYL/IMn8/rv8Am5V36P7urvjza7FgWeScDsdqeKGQ5rVQiqqr3ZmWWopZG/c+z24Su8epkYIWHoNCIeO+mhot2uyxmH3M1st3CHGcMtd6q7bcMbrW3S235kbXzR1oe95lex3R4d2rw5hOiHa34LuyPg9k2c8Nr1jGT52LhWXCqpJ47hSWeOmZSNgnim5WRc7tlxi8Xudou3rQ5TrCKZYGRW7AL7huSVWeZFcK/iRkcVH6LoaOz0FPQtpqZ8jXyckck4a57nNaXPdJvTdAeSnGZtkOUtmtMGGZTiVRVwyRRXytbbpYqJ5YeWQsZVPLtHWhykE6302tBRLDB5PcuyXyLKK7I8ulrspvM1vqYbxa7dHQiinoi51PMyLmeHPBeeYuJ2Og5VYLtwdyPNsAyXGc0zn04LtFDHBPR2iKjZRujfzh/JzvMhLgwuDnaIboBuytYRMsDDLr7na/5PVZNX37PGXC437F6nGJnRWZsEEEbyTHJGwSk7a50hcHOdzc3QsACt2c8IPfdjGM0lJe5rJf8alhqbXeoIGyGKVkZidzROOnsexzg5mxvfj0WiomWBllVwkyPIYMZfk2aR3iusmRQ31s0NobTRuZHBJF3drBIS0EyF/O5zzvY1rWrJjHD73ucQM0yfv/AHj3yOoj3XseXu/d4ey++5jz83j4DXh18Vb0S0Aoyw/4x7l+qab99MpNRlh/xj3L9U0376Zen/XX3ecNR2rsiIvlMiIiAuC/w95sVyi7sys7Smkb3eR/I2XbSOQu/FB8N+W13qPyKE1OP3OIUrK4yUsrRSySdm2bbCOQu/FB8N+W0H8YvT90xm0Qd0joOypIWd0ik7RkGmAcjX/jBvgD562pNRuN03c8dtUHdGUHZUsTO6Rydo2HTAOQO/GDfDfnpSSAiIgIiICzmGobg5q6G4xTsp3VU9RT1cUD5Y5GSyvk5SWNPK5vMWkO8dAje+mjIujCxfZ3iYvErEs79/8AY/ymb2Sb6ie/+x/lM3sk31FoiLo1jC5J6x+K7Gd+/wDsf5TN7JN9RPf/AGP8pm9km+otERNYwuSesfibGd+/+x/lM3sk31E9/wDY/wApm9km+otERNYwuSesfibGbQ8ScdqJJ44q90kkDxHK1lNKTG4tDg1w5eh5XNOj5OB819ff/Y/ymb2Sb6i/rho1rc54s6J5jklO4g6+Z7cP/JaEmsYXJPWPxNjO/f8A2P8AKZvZJvqJ7/7H+UzeyTfUWiImsYXJPWPxNjO/f/Y/ymb2Sb6ie/8Asf5TN7JN9RaIiaxhck9Y/E2M7GfWRx0Kidx+QUcxJ/8A4KWxGhnq75cb7LBJSwT08VJTRTtLJHNY6Rxkc0jbQ4yAAHrpuyBvStyLFePTNM00U2v8b+UJeOwREXEgiIgKJyyA1WK3mEUbLiZKKZgo5ZuxZPthHZl/4gd4c3lvallX+IVKa7AsjpW2uO9unt1RCLZLU92ZV80bm9k6X/Jh2+Uv8t78kEnZaYUVmoKdtO2lbFTxxiBj+dsYDQOUO8wPDfnpdq/iKJkETI428rGNDWtHkB4L+0BERAREQEREBERAREQEREGe4Me68U+JdIQGumqLfcNbGy19I2EHXj40rh1+T8y0JZ7dCMa412mvkJZSZLa3Wlzyfi96pnPqKdmvldFLWHe/8kB1300JAREQEREBERAREQEREBV3P6L0pjE1CbRHfI6uenp5aGapEDXRPnY2R5d/+RhdJyjq7k5R1IViVcyGnZdsisNDJb6WuggldcJJJqgNfTOjbqJ7Ix1eS556noOU+ekFjREQEREBERAREQEREBERAREQQObYnFmePy2987qOobJHVUlbG3mfTVETxJFKB58r2jbfBw209CV8MIyyTIqOaluUDLfkduLYrnb2OJETyDyyRk9XwyaLmP8AMbB09r2tsqrmV4k69S01yttU21ZFRb7rcBEHgsPV0Ezehkhd05mbB2A5pa5rXALGireJ5iL9JPbrjR+hsko2h1ZanyiTlaSQ2aJ+h2sDyDyyaB6Frmse17G2RAREQEREBERARFy3K50lnopKutnZTU0eg6SQ6GyQ1oHykkgADqSQB1KBdbpR2O21Nwr6mOjoqaMyzTzO5WsaBsklRuPW6R1VW3iupqJlxrD2TJqaF7ZO5se808chfpxcBI9xbpoa6RwAOi539UdLXXO4srq8TUDKSaojp6KGoDo52HTWTS6HV2g8tZzcoEm3AvA5JpAREQEREBERAREQEREBERAREQERUCuaMvyC8wVr5Db7ZUNpIqWORzGvf2UcrpH8pHMfugaAegDT8q98LC9rM3m0QsQ5PdA18eLcNrrl0NrudyvVggdU2/0JFz1jZCWgtHQ7iPxTKCC3ka5xB5QvJPuGfdcZXxQ48ZVY86uL5Jsgh73bqL4zIKKSEdYIYyTyNMeyfNxj24ucSV67+D7Hvm1n9o/6Vxy8JMPnrY6yTH6OSrjO2VDmkyN/Q7ewunV8LnnpH5Lsaiizr4Pse+bWf2j/AKU+D7Hvm1n9o/6U1fC556R+RsaKizr4Pse+bWf2j/pT4Pse+bWf2j/pTV8LnnpH5GxoqLOvg+x75tZ/aP8ApT4Pse+bWf2j/pTV8LnnpH5GxQvdycaKjgvwEutXa6x9DkF2e22W+eF3LJE5+zJI0+ILWB2nDqHFpVT9wvx3uvHTFBBlNHeherDQwUXeZ4Xvtte1jtiq7Ut61RIjDxI933gfEG88oGv13CfELm+F1ZYKSrdCSYzOC8sJ8S3Z6eA8PkXS3h5jrGhrbYxrQNAB79D/AHpq+Fzz0j8jY0ZFm9TilLZKOessnPbbjDG6SGRk0hjc4DYEjObT2nWiD5eBB0RerFdG3uyW+4sZ2baynjqAwnfKHtDtf714YuDFERVTN46eqWdyIi5kEREBERAREQEREBERAREQFQLD/L+X/rYf8rTq/qgWH+X8v/Ww/wCVp13aLur7vOFjtTiIs9z3j/gfDO4VFDkN97pV00LaioihpJ6jsGO3yGV0bHCPmI00PI5vLa9JmI3o0JFRsw424ZgVTS017u76WrqKcVYpoqKonmjh/wDxJWRxudEzYI5nho2CPIrmvfH/AAGwVFNBU39s89Tb47rBHb6SesdNSSF4bOwQsfzM3G7ZH3vQu0HDa8DQkVKvnGfDMex6zXurvkb7feWh9udRwyVUlW0tDtxxRNc9wAIJIb0310uOr4/YBRWGz3qXI4TbbvUSUdFNHDK8yzsa5zoS1rC5sg5HDkcA4u00DmIBXgaCize/e6JwHFo6B93vFTbm1tMKyPvFqrGmOEuc0STDstwt21w3Jy+BXFmfugrPhvE/GMTnpquqgvNvmr/SFFRVNU1oa6NsQaIYnh4dzuJcDpga3m0HtKl4GqoqBlPHnA8Lv01nvOQR0ldByd51TzSRUnPrl7eVjDHDsEH7o5vQg+CvzHtkY17HBzXDYcDsEK3HwuP8n1X+qd/wK7OH34BY3+rKb901cdx/k+q/1Tv+BXZw+/ALG/1ZTfumqYvuf38pXsT6Ii+cgiIgIiICIiAiIgIiICIiAqBYf5fy/wDWw/5WnV/VAsP8v5f+th/ytOu7Rd1fd5wsdqcXkDjjT3OyZ1xRtclPc6LDMwpKN94vXvbrK8UrWU4ildBJA1zOURMG+0ILHcxAI1v1+s+y/gHgmeX6a83yyOrK+djY53NrqiGOoY0aa2WNkjWSADppzT0WqovGxGI3jHIbHxYyHIqu3Z5fMUyW3W2eyXHB62uDQ2KnEZhnZTSMI30ka945fujuo6q8cMOH8eGcaqZtosNxtOMw4LSUtP30PkEMprqiV0DpXFwMjQ8Et5iRseWluVLSw0NNDT08TIKeFgjjijaGtY0DQaAOgAHTS+qRTYeJsU4c3vGbRwuv9/sGZvstJYq6zVdHjUlZS3G3TOrTLHI+KBzJXRvY0NIAIGmEjoFo1Jw8oqe88Ja/H8YyOjoJ8qrrxcmX7t6mqhkNDPEJ6h0j5HR8xZERzOHVzdgOJC9JopFEQPNfHelyfIMxyey1tFmNZYqixNhx6kxYSR0tTVyNlbN3yaMtDdHsgGyuEfKXdCSVz203jDTwJymrxfIa6jteKz2a5U1DbZZqyjqXw0vL2kAHOGl0EjebWgdEnRBXpxFcvaPIowSKx5XxBs+X41xFvMOQ3upuFHLjFbXC3VtJUhuopmxTMijewbjcJQ3bWjqQvV9ptdPY7VRW6jYY6SjhZTwsc4uLWMaGtBJ6noB1PVdaKxFhz3H+T6r/AFTv+BXZw+/ALG/1ZTfumrjuP8n1X+qd/wACuzh9+AWN/qym/dNTF9z+/lK9ifREXzkEREBERAREQEREBERAREQFQbKOyyXL4ndJDcmShp82OpYAHfo21w/S0/Ir8oa+YpR32aOoe+opKyNvI2qo5THJy73ykjo4b66IOuvyldOBiU0TMVbpjziVhyouP4OW+sV99pZ9RPg5b6xX32ln1F1ZsHn8JLRxdiLj+DlvrFffaWfUT4OW+sV99pZ9RM2Dz+Elo4uxFx/By31ivvtLPqKpcLseq8txAXG4ZFeO8m4XCm+41DA3khrJoY/xD15I27/PtM2Dz+Elo4ryi4/g5b6xX32ln1E+DlvrFffaWfUTNg8/hJaOLsRcfwct9Yr77Sz6ifBy31ivvtLPqJmwefwktHF+Xupio7NX1EzxHDFTySPe7wa0NJJP9ClsKpJaDDbDTTsMc8NBBHIxw0WuEbQQf6QuGj4fUUNRHLV11yugjcHshranmi5gQQSxoAdogEcwOj18QFaF442JRNEUUbdtzssIiLiQREQEREBERAREQEREBERAREQEREBERAWe8BwBw6bynY9L3f8A+pVP5z/18i0JZ9wI38Hbd8u/S93+91r+Uqn5P+vl6oNBREQEREBERAREQEREBERAREQEREBERAREQEREBERAREQFnnAUBvDkAODv8MXjqP1nUrQ1nnAUAcORo7/wxeP/AKnVINDREQEREBERAREQEREBERAREQEREBERAREQEREBFxXi80Ngt0tdcaqOjpIgOaWU6GydAD5SToADqSQB1KzC6cd5XSubZ7C6aEeE9wqOw5uviGNa46/0uU/mXZo+h4+le6pv4R1lbNcRYieN+Sb6Wi16/nEn1V+fDfknzRavaJPqrv8A7PpnLHWC3xWj3QXFW58FeGFyzG240MqbbXMfVUQre6uZATp0od2cm+UluxrwJO+nXC/cLe6greNkt5xuPCnWe22oVVxlu/pDtmmWprHysg5BCwA6kk+NzHfZb18bpoV24tXm+2qsttfYLRVUFZC+nqIJKiXlkje0tc0/F8CCR/Ss49z1apPc5YdVWGw22grO91klZPWVMzxLIT0Y06Z4NaAB/SfNP7PpnLHWC3xewEWIfDfknzRavaJPqp8N+SfNFq9ok+qn9n0zljrBb4tvRYvS8dLzE8Grx+jmj8+7Vrmv/oDo9H+khaJiOf2nM2yMo5JIayIc0tHUs5JWDetjyc3qPjNJHXW99FyY/wDT9J0anNiU7OO/7FlkREXz0EREBERAREQEREBERAREQEREBEUFndVNQ4RkNTTuLZ4bdUSRuHiHCJxH+8LdFOeqKY7Vjaw7MMvlzm9PrBITaoHuZb4gdsLRsdv+dzxsg+TSB0JduHXyo42w0kEbNcjWNa3XyAdF9V+o4eFRg0Rh0RaIYmbyIizvjHm12xensFtsUMz7pe640rJaeKKWWJjY3SPcxsr2ML9N0OZ2upOjrRYlcYdM1SjREWC3HMuI1gxa6urBV0Tm3C2Q2653ekpBNIJqhsc0ckUEjmEAEaI5CQ8+BG125FxGyLhhU5nRV1x98z6G00tyt81VBHC5kk074OR/ZNaCwPDXb0DrY2fFc2t0RtqiY/k+itkluVHDXwUMlVBHWzsdJFTOkAkka3XM5rd7IHM3ZHhsfKulYtQ2S/2bjfhwv+RnIZ5bPcXB3c46dsLuan5g3kA207GubZGvE76bSvfCxJxM14tafKJQSN81NVU9XSzOpa2mf2kFRH99G7w/pBGwQehBIPQoi9ZiJi0rueh8FyluYYzSXEsbDUEGOphadiOZp09o/NvqPzEKfWUcA5X9hkkP+SbWRyD/AEnQtDv9zW/7Vq6/NtNwacDSK8OndE/fa3IiIuFBERAREQEREBERAREQEREBfKqpoqymlp5mCSGVhjew+DmkaI/2L6om7bA8tTWmox2rns9Zs1NC7sS93jKwfeSfoc3R/TseSgchr8jpJ4m2SzW+5wlu5H1tzfSlrt+ADYJNjXnsfoXpjOuH9HmkEcnP3K6QDUNY1nMdf+B46czN9dbGj1BCxy64NlFkkc2osc1ZGPCotrhOx/6G9Hj+lv8ASV/f6L/UcLSqIiqrLX27o6X2ExfbDNjec85R/wDZSx82zse+CXWvL/4T9K5rnitfxHtxo8rtUFkdSzR1VBW2a7PmqIZ27+6NcYY+QgHX4wPMdhX91uu7SQcevex06WyY/wD+V+ej7t6vXz9lz/VXf/hVsqrvH7eiZZ4KNJwrpazHX2i4Xy93Vr66CvdVVtSx83PE9j2NGmBrWbjGw1o8T5na6b9wxsmTXa711yZNVC62tloqaZzwIjC173gt0OYP3Ieu/IaAI2rh6Pu3q9fP2XP9VPR929Xr5+y5/qq5cDdMx1/nEyzwZtb+FDsYuVLfaS73jJbxbqSWkooL5cGiLkkLNtL2wkj7wfG0SfPfTUmL1nvnidj/AP3BL/Bq7ej7t6vXz9lz/VT0fdvV6+fsuf6qkRh07KK7fvHncyzwU6ku+byVcLanGLLDTOe0Syx32R7mN31IaaUcxA302N/KFappmU8L5ZXiONjS5z3HQaB1JK76XHsguD+Smxq6vefDtqfu7f60paFpGD8IZKOrhuWQuhmnhdzwW+E88THeT5HEDncPEAANaevxiGkc+NpmDotMzXXmnhsv4eZl4pvhFjVRjuKdpWxvhrrhM6slik++iBAaxh+QhjW7Hk4uV3RF+f42LVj4lWJVvlREReIIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIg//Z\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import Image, display\n",
    "\n",
    "try:\n",
    "    display(Image(app.get_graph(xray=True).draw_mermaid_png()))\n",
    "except:\n",
    "    # This requires some extra dependencies and is optional\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5344cab3-d2e4-4227-822d-124cf87b57d7",
   "metadata": {},
   "source": [
    "## 测试一下"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30d04898-3c5a-4b63-a9c5-6086bad9a002",
   "metadata": {},
   "source": [
    "模拟答案在知识库内的情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c73b82a4-e97d-4318-8b1a-749a7c379713",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:14.605419Z",
     "iopub.status.busy": "2024-11-27T11:21:14.605270Z",
     "iopub.status.idle": "2024-11-27T11:21:21.669118Z",
     "shell.execute_reply": "2024-11-27T11:21:21.668580Z",
     "shell.execute_reply.started": "2024-11-27T11:21:14.605407Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---RETRIEVE---\n",
      "\"Output from node 'retrieve':\"\n",
      "'---'\n",
      "{ 'documents': [ Document(metadata={'page': 50, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'edf68bd0-d507-4739-a4d4-b2e87215b655'}, page_content='全球经济金融展望报告\\n中国银行研究院 49 2024年\\n场仍处于下行周期的早期阶段。基于历史经验，叠加长期结构性因素，商业房\\n地产的此轮下行周期可能持续更长时间。第四，美国大型银行更具有能力抵御\\n严重衰退和房地产市场萎缩的冲击，而中小型银行面临的损失可能更大。'),\n",
      "                 Document(metadata={'page': 33, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': '3e312dfa-dd43-4ab9-961f-a2e442c89cdd'}, page_content='全球经济金融展望报告\\n中国银行研究院 32 2024年\\n图19：美国联邦基金目标利率与全球MSCI指数\\n资料来源：Wind，中国银行研究院\\n表3：全球主要股指概览\\n注：涨跌幅区间为2023年1月1日至2023年11月15日，收盘价和市盈\\n率为2023年11月15日。\\n资料来源：Wind，中国银行研究院'),\n",
      "                 Document(metadata={'page': 0, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'e73a0c9d-d42b-4350-a4c3-b38bf67c68a5'}, page_content='研究院\\n全球经济金融展望报告\\n要点2024年年报（总第57期） 报告日期：2023年12月12日\\n●2023年全球经济增长动力持续回落，各国复苏分化，\\n发达经济体增速明显放缓，新兴经济体整体表现稳定。\\n全球贸易增长乏力，各国生产景气度逐渐回落，内需\\n对经济的拉动作用减弱。欧美央行货币政策紧缩态势\\n放缓，美元指数高位震荡后走弱，全球股市表现总体\\n好于预期，但区域分化明显。高利率环境抑制债券融\\n资需求，债券违约风险持续上升。\\n●展望2024年，预计全球经济复苏将依旧疲软，主要\\n经济体增长态势和货币政策走势将进一步分化。欧美\\n央行大概率结束本轮紧缩货币周期，美元指数将逐步\\n走弱，流向新兴经济体的跨境资本将增加。国际原油\\n市场短缺格局或延续，新能源发展成为重点。\\n●海湾六国经济发展与投资前景、高利率和高债务对\\n美国房地产市场脆弱性的影响等热点问题值得关注。中国银行研究院\\n全球经济金融研究课题组\\n组长：陈卫东\\n副组长：钟红\\n廖淑萍\\n成员：边卫红\\n熊启跃\\n王有鑫\\n曹鸿宇\\n李颖婷\\n王宁远\\n初晓\\n章凯莉\\n黄小军（纽约）\\n陆晓明（纽约）\\n黄承煜（纽约）\\n宋达志（伦敦）\\n李振龙（伦敦）\\n张传捷（伦敦）\\n刘冰彦（法兰克福）\\n温颍坤（法兰克福）\\n张明捷（法兰克福）\\n王哲（东京）\\n李彧（香港）\\n黎永康（香港）\\n联系人：王有鑫\\n电话：010-66594127\\n邮件：wangyouxin_hq@bank-of-china.com主要经济体GDP增速变化趋势（%）\\n资料来源：IMF，中国银行研究院')],\n",
      "  'question': '报告的时间是什么时候?'}\n",
      "'\\n---\\n'\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: GENERATE---\n",
      "\"Output from node 'grade_documents':\"\n",
      "'---'\n",
      "{ 'documents': [ Document(metadata={'page': 50, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'edf68bd0-d507-4739-a4d4-b2e87215b655'}, page_content='全球经济金融展望报告\\n中国银行研究院 49 2024年\\n场仍处于下行周期的早期阶段。基于历史经验，叠加长期结构性因素，商业房\\n地产的此轮下行周期可能持续更长时间。第四，美国大型银行更具有能力抵御\\n严重衰退和房地产市场萎缩的冲击，而中小型银行面临的损失可能更大。'),\n",
      "                 Document(metadata={'page': 33, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': '3e312dfa-dd43-4ab9-961f-a2e442c89cdd'}, page_content='全球经济金融展望报告\\n中国银行研究院 32 2024年\\n图19：美国联邦基金目标利率与全球MSCI指数\\n资料来源：Wind，中国银行研究院\\n表3：全球主要股指概览\\n注：涨跌幅区间为2023年1月1日至2023年11月15日，收盘价和市盈\\n率为2023年11月15日。\\n资料来源：Wind，中国银行研究院'),\n",
      "                 Document(metadata={'page': 0, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'e73a0c9d-d42b-4350-a4c3-b38bf67c68a5'}, page_content='研究院\\n全球经济金融展望报告\\n要点2024年年报（总第57期） 报告日期：2023年12月12日\\n●2023年全球经济增长动力持续回落，各国复苏分化，\\n发达经济体增速明显放缓，新兴经济体整体表现稳定。\\n全球贸易增长乏力，各国生产景气度逐渐回落，内需\\n对经济的拉动作用减弱。欧美央行货币政策紧缩态势\\n放缓，美元指数高位震荡后走弱，全球股市表现总体\\n好于预期，但区域分化明显。高利率环境抑制债券融\\n资需求，债券违约风险持续上升。\\n●展望2024年，预计全球经济复苏将依旧疲软，主要\\n经济体增长态势和货币政策走势将进一步分化。欧美\\n央行大概率结束本轮紧缩货币周期，美元指数将逐步\\n走弱，流向新兴经济体的跨境资本将增加。国际原油\\n市场短缺格局或延续，新能源发展成为重点。\\n●海湾六国经济发展与投资前景、高利率和高债务对\\n美国房地产市场脆弱性的影响等热点问题值得关注。中国银行研究院\\n全球经济金融研究课题组\\n组长：陈卫东\\n副组长：钟红\\n廖淑萍\\n成员：边卫红\\n熊启跃\\n王有鑫\\n曹鸿宇\\n李颖婷\\n王宁远\\n初晓\\n章凯莉\\n黄小军（纽约）\\n陆晓明（纽约）\\n黄承煜（纽约）\\n宋达志（伦敦）\\n李振龙（伦敦）\\n张传捷（伦敦）\\n刘冰彦（法兰克福）\\n温颍坤（法兰克福）\\n张明捷（法兰克福）\\n王哲（东京）\\n李彧（香港）\\n黎永康（香港）\\n联系人：王有鑫\\n电话：010-66594127\\n邮件：wangyouxin_hq@bank-of-china.com主要经济体GDP增速变化趋势（%）\\n资料来源：IMF，中国银行研究院')],\n",
      "  'question': '报告的时间是什么时候?',\n",
      "  'run_web_search': 'No'}\n",
      "'\\n---\\n'\n",
      "---GENERATE---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3999598/3884634306.py:18: LangChainDeprecationWarning: The class `Ollama` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the :class:`~langchain-ollama package and should be used instead. To use it run `pip install -U :class:`~langchain-ollama` and import as `from :class:`~langchain_ollama import OllamaLLM``.\n",
      "  llm = Ollama(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"Output from node 'generate':\"\n",
      "'---'\n",
      "{ 'documents': [ Document(metadata={'page': 50, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'edf68bd0-d507-4739-a4d4-b2e87215b655'}, page_content='全球经济金融展望报告\\n中国银行研究院 49 2024年\\n场仍处于下行周期的早期阶段。基于历史经验，叠加长期结构性因素，商业房\\n地产的此轮下行周期可能持续更长时间。第四，美国大型银行更具有能力抵御\\n严重衰退和房地产市场萎缩的冲击，而中小型银行面临的损失可能更大。'),\n",
      "                 Document(metadata={'page': 33, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': '3e312dfa-dd43-4ab9-961f-a2e442c89cdd'}, page_content='全球经济金融展望报告\\n中国银行研究院 32 2024年\\n图19：美国联邦基金目标利率与全球MSCI指数\\n资料来源：Wind，中国银行研究院\\n表3：全球主要股指概览\\n注：涨跌幅区间为2023年1月1日至2023年11月15日，收盘价和市盈\\n率为2023年11月15日。\\n资料来源：Wind，中国银行研究院'),\n",
      "                 Document(metadata={'page': 0, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'e73a0c9d-d42b-4350-a4c3-b38bf67c68a5'}, page_content='研究院\\n全球经济金融展望报告\\n要点2024年年报（总第57期） 报告日期：2023年12月12日\\n●2023年全球经济增长动力持续回落，各国复苏分化，\\n发达经济体增速明显放缓，新兴经济体整体表现稳定。\\n全球贸易增长乏力，各国生产景气度逐渐回落，内需\\n对经济的拉动作用减弱。欧美央行货币政策紧缩态势\\n放缓，美元指数高位震荡后走弱，全球股市表现总体\\n好于预期，但区域分化明显。高利率环境抑制债券融\\n资需求，债券违约风险持续上升。\\n●展望2024年，预计全球经济复苏将依旧疲软，主要\\n经济体增长态势和货币政策走势将进一步分化。欧美\\n央行大概率结束本轮紧缩货币周期，美元指数将逐步\\n走弱，流向新兴经济体的跨境资本将增加。国际原油\\n市场短缺格局或延续，新能源发展成为重点。\\n●海湾六国经济发展与投资前景、高利率和高债务对\\n美国房地产市场脆弱性的影响等热点问题值得关注。中国银行研究院\\n全球经济金融研究课题组\\n组长：陈卫东\\n副组长：钟红\\n廖淑萍\\n成员：边卫红\\n熊启跃\\n王有鑫\\n曹鸿宇\\n李颖婷\\n王宁远\\n初晓\\n章凯莉\\n黄小军（纽约）\\n陆晓明（纽约）\\n黄承煜（纽约）\\n宋达志（伦敦）\\n李振龙（伦敦）\\n张传捷（伦敦）\\n刘冰彦（法兰克福）\\n温颍坤（法兰克福）\\n张明捷（法兰克福）\\n王哲（东京）\\n李彧（香港）\\n黎永康（香港）\\n联系人：王有鑫\\n电话：010-66594127\\n邮件：wangyouxin_hq@bank-of-china.com主要经济体GDP增速变化趋势（%）\\n资料来源：IMF，中国银行研究院')],\n",
      "  'generation': '2023年12月12日',\n",
      "  'question': '报告的时间是什么时候?'}\n",
      "'\\n---\\n'\n"
     ]
    }
   ],
   "source": [
    "inputs = {\"keys\": {\"question\": \"报告的时间是什么时候?\"}}\n",
    "for output in app.stream(inputs):\n",
    "    for key, value in output.items():\n",
    "        pprint.pprint(f\"Output from node '{key}':\")\n",
    "        pprint.pprint(\"---\")\n",
    "        pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
    "    pprint.pprint(\"\\n---\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c179a47b-7f08-4db7-a484-a66b987b657f",
   "metadata": {},
   "source": [
    "查看最终答案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d1e1c05b-a864-47fd-87f9-da9e59164273",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:21.669900Z",
     "iopub.status.busy": "2024-11-27T11:21:21.669603Z",
     "iopub.status.idle": "2024-11-27T11:21:21.672952Z",
     "shell.execute_reply": "2024-11-27T11:21:21.672556Z",
     "shell.execute_reply.started": "2024-11-27T11:21:21.669886Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2023年12月12日'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output['generate']['keys']['generation']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c4746c3-90d1-4f51-b675-8d90fdc663ea",
   "metadata": {},
   "source": [
    "模拟答案不在知识库范围的情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a0bf02a7-2fbf-472a-94ee-89b97d7a9dcc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:21.673441Z",
     "iopub.status.busy": "2024-11-27T11:21:21.673308Z",
     "iopub.status.idle": "2024-11-27T11:21:21.686493Z",
     "shell.execute_reply": "2024-11-27T11:21:21.685933Z",
     "shell.execute_reply.started": "2024-11-27T11:21:21.673428Z"
    }
   },
   "outputs": [],
   "source": [
    "def demo():\n",
    "    inputs = {\"keys\": {\"question\": \"2024年美国总统大选结果是什么，谁赢得了竞选?\"}}\n",
    "    for output in app.stream(inputs):\n",
    "        for key, value in output.items():\n",
    "            pprint.pprint(f\"Output from node '{key}':\")\n",
    "            pprint.pprint(\"---\")\n",
    "            pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
    "        pprint.pprint(\"\\n---\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ed76075-5ac3-417c-bd8f-f5fdf4337214",
   "metadata": {},
   "source": [
    "这部分不稳定，加个重试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c9cf5749-ea82-480d-9d69-14ae90f73241",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:21.687230Z",
     "iopub.status.busy": "2024-11-27T11:21:21.687056Z",
     "iopub.status.idle": "2024-11-27T11:21:42.273828Z",
     "shell.execute_reply": "2024-11-27T11:21:42.273301Z",
     "shell.execute_reply.started": "2024-11-27T11:21:21.687213Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---RETRIEVE---\n",
      "\"Output from node 'retrieve':\"\n",
      "'---'\n",
      "{ 'documents': [ Document(metadata={'page': 34, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'dc18ddce-a123-45b5-86e2-2ff1378fb6aa'}, page_content='全球经济金融展望报告\\n中国银行研究院 33 2024年\\n展望2024年，全球股票市场走势将呈现以下特征。\\n1.美国股票市场上行空间有限，新兴经济体股市值得关注\\n2024年，受美国通胀回落、美联储暂停加息以及企业盈利能力稳步提升等\\n因素支撑，美国股票市场将呈现上扬态势。但考虑到美股股价虚高等因素，后\\n续美股上行空间相对有限。新兴经济体稳步复苏，未来反弹空间值得期待。\\n2.科技股仍是投资热点，绿色能源受到投资青睐\\n2024年，全球股市将延续由科技股领涨的结构性行情。截至2023年11月\\n15日，信息科技MSCI指数涨幅高达41.1%，远高于电信业务（31.5%）、非日\\n常消费品（21.5%）等板块（图20）。其中，以人工智能、大数据等新兴行业为\\n代表的科技公司或将受到广泛关注。随着企业盈利能力逐渐增强以及科技水平\\n不断发展，科技股依然是投资者热衷的板块。此外，在全球投资者绿色环保意\\n识显著增强的背景下，绿色能源板块崛起或成为全球股市投资者关注的亮点。\\n图20：全球MSCI一级行业指数涨跌幅（%）\\n注：涨跌幅区间为2023年1月1日至2023年11月15日，收盘价和市盈\\n率为2023年11月15日。\\n资料来源：Wind，中国银行研究院'),\n",
      "                 Document(metadata={'page': 11, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': '0943b71a-7314-4cef-a7d9-682c7ddc9d7f'}, page_content='全球经济金融展望报告\\n中国银行研究院 10 2024年\\n模下降2.5%，进口降幅大于出口，带动贸易赤字收窄12.1%。\\n图8：各分项对美国GDP环比增长折年率的季度拉动率变化趋势（%）\\n资料来源：Wind，中国银行研究院\\n2023年四季度美国经济增长动力明显回落，更多领域的负面因素开始凸显，\\n不同部门表现分化。一是对经济增长贡献最大的私人消费表现疲软。消费者支\\n出增幅有所放缓，10月零售和食品销售环比增速由之前两个月的0.7%以上降至\\n-0.1%，私人消费对经济支撑作用减弱。二是制造业投资和工业产出处于疲弱状\\n态。三季度企业制造业投资在连续7个季度增长后转为下降，未来可能会进一\\n步回落。ISM制造业PMI指数长期处于萎缩区间，10月数值较上个月大幅下降\\n2.3个百分点。三是美国就业市场也逐渐降温，10月新增非农就业人数较年内\\n月度均值大幅减少，失业率提高至3.9%，工资增速放缓，在通胀高企背景下，\\n居民可支配收入增长逐渐承压。四季度美国GDP环比增长折年率可能会回落至\\n2%左右。\\n预计2024年美国经济增速将进一步回落，“软着陆”和衰退的可能性同时\\n存在，但从目前情况看，“软着陆”发生的可能性要高于衰退。货币政策是影'),\n",
      "                 Document(metadata={'page': 12, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': '2800873d-c399-4c23-9721-9238e2c941f3'}, page_content='全球经济金融展望报告\\n中国银行研究院 11 2024年\\n响2024年美国经济走势的重要因素。2023年美国通胀已明显放缓，受基期效\\n应减弱、住宅成本居高不下、全球地缘政治动荡加剧可能导致食品与能源价格\\n反弹等因素影响，2024年美国通胀进一步回落至2%目标将遇到一定阻力，后\\n续下行难度增大且具有反弹风险。当前，美联储货币政策路线尚不清晰，但市\\n场基本达成几点共识：第一，当前的货币政策已经达到“限制性”水平；第二，\\n美联储加息周期已临近尾声，大概率在2024年调整货币政策；第三，若无特殊\\n风险事件发生，美联储可能在2024年维持当前缩表（QT）节奏。目前不确定\\n的是高利率的持续时间与首次降息时点，以及潜在的货币政策“滞后效应”可\\n能对经济带来的影响。\\n对于私人消费来说，高利率对消费的负面影响将逐渐显现。2023年利率上\\n升增加了美国家庭债务负担，家庭债务支出/可支配收入虽低于历史水平，但呈\\n现上升态势。美国家庭债务余额拖欠率回升至3%，消费者借贷能力和意愿将有\\n所下降，债务融资在消费者支出来源中的权重上升。叠加学生贷款暂停还款阶\\n段结束以及疫情时期放宽福利资格计划结束，将使得消费者财务状况恶化并影\\n响消费支出的可持续性。2024年随着美国劳动力市场紧张状况缓解，其对冲加\\n息负面影响的调节效应将逐渐减弱。具有先导性的首次失业保险申请人数缓慢\\n回升，就业增长可能进一步减速。堪萨斯联储劳工市场状态指数中的动能指数\\n低于历史平均值，反映未来增长的乏力状态。未来劳动者收入增幅可能下降，\\n收入对支出的支撑作用可能减弱。\\n对于私人投资来说，金融条件虽仍相对宽松，但信贷条件逐渐收紧，银行\\n业普遍提高了对各种规模企业的信贷标准，NFIB小企业乐观指数显示小企业信\\n心仍然疲软。此外，耐用品订单持续疲弱，长期高利率和紧缩的金融条件迫使\\n企业缩减投资。9月美国二手房销售额跌至十年来最低水平，未来在价格持续\\n上升、房贷利率维持高位的背景下，住房负担能力将继续处于历史低位，销售\\n预期可能持续遇冷。但另一方面，房屋供应量保持低位，可能促使建筑相关投')],\n",
      "  'question': '2024年美国总统大选结果是什么，谁赢得了竞选?'}\n",
      "'\\n---\\n'\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT NOT RELEVANT---\n",
      "error: list index out of range, max_retries=2\n",
      "---RETRIEVE---\n",
      "\"Output from node 'retrieve':\"\n",
      "'---'\n",
      "{ 'documents': [ Document(metadata={'page': 34, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': 'dc18ddce-a123-45b5-86e2-2ff1378fb6aa'}, page_content='全球经济金融展望报告\\n中国银行研究院 33 2024年\\n展望2024年，全球股票市场走势将呈现以下特征。\\n1.美国股票市场上行空间有限，新兴经济体股市值得关注\\n2024年，受美国通胀回落、美联储暂停加息以及企业盈利能力稳步提升等\\n因素支撑，美国股票市场将呈现上扬态势。但考虑到美股股价虚高等因素，后\\n续美股上行空间相对有限。新兴经济体稳步复苏，未来反弹空间值得期待。\\n2.科技股仍是投资热点，绿色能源受到投资青睐\\n2024年，全球股市将延续由科技股领涨的结构性行情。截至2023年11月\\n15日，信息科技MSCI指数涨幅高达41.1%，远高于电信业务（31.5%）、非日\\n常消费品（21.5%）等板块（图20）。其中，以人工智能、大数据等新兴行业为\\n代表的科技公司或将受到广泛关注。随着企业盈利能力逐渐增强以及科技水平\\n不断发展，科技股依然是投资者热衷的板块。此外，在全球投资者绿色环保意\\n识显著增强的背景下，绿色能源板块崛起或成为全球股市投资者关注的亮点。\\n图20：全球MSCI一级行业指数涨跌幅（%）\\n注：涨跌幅区间为2023年1月1日至2023年11月15日，收盘价和市盈\\n率为2023年11月15日。\\n资料来源：Wind，中国银行研究院'),\n",
      "                 Document(metadata={'page': 11, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': '0943b71a-7314-4cef-a7d9-682c7ddc9d7f'}, page_content='全球经济金融展望报告\\n中国银行研究院 10 2024年\\n模下降2.5%，进口降幅大于出口，带动贸易赤字收窄12.1%。\\n图8：各分项对美国GDP环比增长折年率的季度拉动率变化趋势（%）\\n资料来源：Wind，中国银行研究院\\n2023年四季度美国经济增长动力明显回落，更多领域的负面因素开始凸显，\\n不同部门表现分化。一是对经济增长贡献最大的私人消费表现疲软。消费者支\\n出增幅有所放缓，10月零售和食品销售环比增速由之前两个月的0.7%以上降至\\n-0.1%，私人消费对经济支撑作用减弱。二是制造业投资和工业产出处于疲弱状\\n态。三季度企业制造业投资在连续7个季度增长后转为下降，未来可能会进一\\n步回落。ISM制造业PMI指数长期处于萎缩区间，10月数值较上个月大幅下降\\n2.3个百分点。三是美国就业市场也逐渐降温，10月新增非农就业人数较年内\\n月度均值大幅减少，失业率提高至3.9%，工资增速放缓，在通胀高企背景下，\\n居民可支配收入增长逐渐承压。四季度美国GDP环比增长折年率可能会回落至\\n2%左右。\\n预计2024年美国经济增速将进一步回落，“软着陆”和衰退的可能性同时\\n存在，但从目前情况看，“软着陆”发生的可能性要高于衰退。货币政策是影'),\n",
      "                 Document(metadata={'page': 12, 'source': 'data/2024全球经济金融展望报告.pdf', 'uuid': '2800873d-c399-4c23-9721-9238e2c941f3'}, page_content='全球经济金融展望报告\\n中国银行研究院 11 2024年\\n响2024年美国经济走势的重要因素。2023年美国通胀已明显放缓，受基期效\\n应减弱、住宅成本居高不下、全球地缘政治动荡加剧可能导致食品与能源价格\\n反弹等因素影响，2024年美国通胀进一步回落至2%目标将遇到一定阻力，后\\n续下行难度增大且具有反弹风险。当前，美联储货币政策路线尚不清晰，但市\\n场基本达成几点共识：第一，当前的货币政策已经达到“限制性”水平；第二，\\n美联储加息周期已临近尾声，大概率在2024年调整货币政策；第三，若无特殊\\n风险事件发生，美联储可能在2024年维持当前缩表（QT）节奏。目前不确定\\n的是高利率的持续时间与首次降息时点，以及潜在的货币政策“滞后效应”可\\n能对经济带来的影响。\\n对于私人消费来说，高利率对消费的负面影响将逐渐显现。2023年利率上\\n升增加了美国家庭债务负担，家庭债务支出/可支配收入虽低于历史水平，但呈\\n现上升态势。美国家庭债务余额拖欠率回升至3%，消费者借贷能力和意愿将有\\n所下降，债务融资在消费者支出来源中的权重上升。叠加学生贷款暂停还款阶\\n段结束以及疫情时期放宽福利资格计划结束，将使得消费者财务状况恶化并影\\n响消费支出的可持续性。2024年随着美国劳动力市场紧张状况缓解，其对冲加\\n息负面影响的调节效应将逐渐减弱。具有先导性的首次失业保险申请人数缓慢\\n回升，就业增长可能进一步减速。堪萨斯联储劳工市场状态指数中的动能指数\\n低于历史平均值，反映未来增长的乏力状态。未来劳动者收入增幅可能下降，\\n收入对支出的支撑作用可能减弱。\\n对于私人投资来说，金融条件虽仍相对宽松，但信贷条件逐渐收紧，银行\\n业普遍提高了对各种规模企业的信贷标准，NFIB小企业乐观指数显示小企业信\\n心仍然疲软。此外，耐用品订单持续疲弱，长期高利率和紧缩的金融条件迫使\\n企业缩减投资。9月美国二手房销售额跌至十年来最低水平，未来在价格持续\\n上升、房贷利率维持高位的背景下，住房负担能力将继续处于历史低位，销售\\n预期可能持续遇冷。但另一方面，房屋供应量保持低位，可能促使建筑相关投')],\n",
      "  'question': '2024年美国总统大选结果是什么，谁赢得了竞选?'}\n",
      "'\\n---\\n'\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT NOT RELEVANT---\n",
      "---GRADE: DOCUMENT NOT RELEVANT---\n",
      "---GRADE: DOCUMENT NOT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: TRANSFORM QUERY and RUN WEB SEARCH---\n",
      "\"Output from node 'grade_documents':\"\n",
      "'---'\n",
      "{ 'documents': [],\n",
      "  'question': '2024年美国总统大选结果是什么，谁赢得了竞选?',\n",
      "  'run_web_search': 'Yes'}\n",
      "'\\n---\\n'\n",
      "---TRANSFORM QUERY---\n",
      "\"Output from node 'transform_query':\"\n",
      "'---'\n",
      "{ 'documents': [],\n",
      "  'question': 'What was the outcome of the 2024 United States Presidential '\n",
      "              'Election, who won the campaign?'}\n",
      "'\\n---\\n'\n",
      "---WEB SEARCH---\n",
      "question: What was the outcome of the 2024 United States Presidential Election, who won the campaign?\n",
      "\"Output from node 'web_search':\"\n",
      "'---'\n",
      "{ 'documents': [ Document(metadata={}, page_content=\"US Presidential Election Results 2024 - BBC News Close menu BBC News Kamala Harris of the Democrat party has 0 electoral college votes. Donald Trump of the Republican party has 0 electoral college votes. Kamala Harris of the Democrat party has 158,810 votes (38.4%) Donald Trump of the Republican party has 249,225 votes (60.2%) US presidential election results 2024 US election 2024 Voting in some states is particularly hard to predict, with polls showing they could be won by the Republicans or the Democratic party. The battleground states that could decide the 2024 presidential election are: Voters in 11 states will also elect a governor. US election polls: Who is ahead - Harris or Trump? US election 2024 About the BBC\\nUS election results: Key takeaways as Trump wins | US Election 2024 News | Al Jazeera US election results: Key takeaways as Trump wins Trump makes inroads into Black voter communities; projected to win key swing states North Carolina, Georgia, Pennsylvania and Wisconsin Former United States President Donald Trump has won the presidency again, defeating Democratic Party nominee, Vice President Kamala Harris in Tuesday’s election, according to the Associated Press. Americas Coverage Newsletter But nationally, this year, Trump secured 20 percent of the Black vote, according to an exit poll by AP. In swing states Michigan and Wisconsin, Trump saw a five percentage point increase compared with 2020 among votes below 45 years, exit polls showed.\\nPresidential election results 2024 | CNN Politics CNN10 About CNN Watch CNN CNN Harris is hoping to become the first woman elected president, while Trump is vying to become only the second former president to win a nonconsecutive second term after Grover Cleveland in 1892. Maine 4 Electoral Votes Harris 0Trump 0 Nebraska 5 Electoral Votes Harris 0Trump 0 What's so important to note here is that Harris' best path to 270 electoral votes in pre-election polls ran through Michigan, Pennsylvania and Wisconsin. Dems may need to hold their seats in Ohio and Montana -- two states Trump won twice before -- along with MI, WI, PA, AZ, NV and MD to keep the Senate at 50-50 and hope Harris wins to give them control of the chamber. CNN10 About CNN\\nThe result of all the tumult has been razor-thin polling margins in the Real Clear Politics average of national surveys, Trump led Harris by 0.1 percentage point, well within the margin of error for each of the surveys included. As Americans from coast to coast make their voices heard at the ballot box, keep up with the USA TODAY Network's live coverage of the 2024 presidential election and check back here for results. Election Day 2024 is here, and Harris and Trump are neck-and-neck in the polls. If you're still undecided between the Democratic and Republican nominees, check out USA TODAY's voter guide to see what Trump and Harris have said about major issues, in their own words.\\nDonald Trump defeats Kamala Harris to become the next U.S. president, NBC News projects BREAKING: Donald Trump is elected 47th president of the United States, NBC News projects Donald Trump defeats Kamala Harris to become the next U.S. president, NBC News projects Donald Trump defeats Kamala Harris to become the next U.S. president, NBC News projects Trump will make history as the 45th and now 47th president, NBC News projects, saying he will fix an ailing country and despite warnings he will rule as an authoritarian. NBC News projects Donald Trump is elected 47th president of the United States NBC News projected the Trump victory over Harris, who was the first woman of color to win a major party nomination for president, early Wednesday morning.\")],\n",
      "  'question': 'What was the outcome of the 2024 United States Presidential '\n",
      "              'Election, who won the campaign?'}\n",
      "'\\n---\\n'\n",
      "---GENERATE---\n",
      "\"Output from node 'generate':\"\n",
      "'---'\n",
      "{ 'documents': [ Document(metadata={}, page_content=\"US Presidential Election Results 2024 - BBC News Close menu BBC News Kamala Harris of the Democrat party has 0 electoral college votes. Donald Trump of the Republican party has 0 electoral college votes. Kamala Harris of the Democrat party has 158,810 votes (38.4%) Donald Trump of the Republican party has 249,225 votes (60.2%) US presidential election results 2024 US election 2024 Voting in some states is particularly hard to predict, with polls showing they could be won by the Republicans or the Democratic party. The battleground states that could decide the 2024 presidential election are: Voters in 11 states will also elect a governor. US election polls: Who is ahead - Harris or Trump? US election 2024 About the BBC\\nUS election results: Key takeaways as Trump wins | US Election 2024 News | Al Jazeera US election results: Key takeaways as Trump wins Trump makes inroads into Black voter communities; projected to win key swing states North Carolina, Georgia, Pennsylvania and Wisconsin Former United States President Donald Trump has won the presidency again, defeating Democratic Party nominee, Vice President Kamala Harris in Tuesday’s election, according to the Associated Press. Americas Coverage Newsletter But nationally, this year, Trump secured 20 percent of the Black vote, according to an exit poll by AP. In swing states Michigan and Wisconsin, Trump saw a five percentage point increase compared with 2020 among votes below 45 years, exit polls showed.\\nPresidential election results 2024 | CNN Politics CNN10 About CNN Watch CNN CNN Harris is hoping to become the first woman elected president, while Trump is vying to become only the second former president to win a nonconsecutive second term after Grover Cleveland in 1892. Maine 4 Electoral Votes Harris 0Trump 0 Nebraska 5 Electoral Votes Harris 0Trump 0 What's so important to note here is that Harris' best path to 270 electoral votes in pre-election polls ran through Michigan, Pennsylvania and Wisconsin. Dems may need to hold their seats in Ohio and Montana -- two states Trump won twice before -- along with MI, WI, PA, AZ, NV and MD to keep the Senate at 50-50 and hope Harris wins to give them control of the chamber. CNN10 About CNN\\nThe result of all the tumult has been razor-thin polling margins in the Real Clear Politics average of national surveys, Trump led Harris by 0.1 percentage point, well within the margin of error for each of the surveys included. As Americans from coast to coast make their voices heard at the ballot box, keep up with the USA TODAY Network's live coverage of the 2024 presidential election and check back here for results. Election Day 2024 is here, and Harris and Trump are neck-and-neck in the polls. If you're still undecided between the Democratic and Republican nominees, check out USA TODAY's voter guide to see what Trump and Harris have said about major issues, in their own words.\\nDonald Trump defeats Kamala Harris to become the next U.S. president, NBC News projects BREAKING: Donald Trump is elected 47th president of the United States, NBC News projects Donald Trump defeats Kamala Harris to become the next U.S. president, NBC News projects Donald Trump defeats Kamala Harris to become the next U.S. president, NBC News projects Trump will make history as the 45th and now 47th president, NBC News projects, saying he will fix an ailing country and despite warnings he will rule as an authoritarian. NBC News projects Donald Trump is elected 47th president of the United States NBC News projected the Trump victory over Harris, who was the first woman of color to win a major party nomination for president, early Wednesday morning.\")],\n",
      "  'generation': 'Donald Trump defeated Kamala Harris and was projected as the '\n",
      "                'winner of the 2024 United States Presidential Election by NBC '\n",
      "                'News.',\n",
      "  'question': 'What was the outcome of the 2024 United States Presidential '\n",
      "              'Election, who won the campaign?'}\n",
      "'\\n---\\n'\n"
     ]
    }
   ],
   "source": [
    "max_retries = 3\n",
    "while max_retries > 0:\n",
    "    try:\n",
    "        demo()\n",
    "        break\n",
    "    except Exception as e:\n",
    "        max_retries -= 1\n",
    "        print(f\"error: {e}, max_retries={max_retries}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1292578-7ea6-40ce-b6f8-039b42fa6a00",
   "metadata": {},
   "source": [
    "如果只想获取答案，可以调用invoke方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7fc52deb-8a65-427e-b028-18c50f541650",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:42.274384Z",
     "iopub.status.busy": "2024-11-27T11:21:42.274256Z",
     "iopub.status.idle": "2024-11-27T11:21:42.276691Z",
     "shell.execute_reply": "2024-11-27T11:21:42.276357Z",
     "shell.execute_reply.started": "2024-11-27T11:21:42.274371Z"
    }
   },
   "outputs": [],
   "source": [
    "# output = app.invoke(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d9f043e-b84d-4779-bbdd-d8a822353407",
   "metadata": {},
   "source": [
    "# 生成答案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "812ccfa4-068d-4184-936f-4fff28ae87e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:55.734291Z",
     "iopub.status.busy": "2024-11-27T11:21:55.733498Z",
     "iopub.status.idle": "2024-11-27T11:21:55.739363Z",
     "shell.execute_reply": "2024-11-27T11:21:55.738863Z",
     "shell.execute_reply.started": "2024-11-27T11:21:55.734219Z"
    }
   },
   "outputs": [],
   "source": [
    "llm = Ollama(\n",
    "    model='qwen2:7b-instruct',\n",
    "    base_url='http://localhost:11434',\n",
    "    top_k=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "05369c9e-0e88-4fb6-ae1b-f897755fbcdd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:56.376532Z",
     "iopub.status.busy": "2024-11-27T11:21:56.376325Z",
     "iopub.status.idle": "2024-11-27T11:21:56.380238Z",
     "shell.execute_reply": "2024-11-27T11:21:56.379812Z",
     "shell.execute_reply.started": "2024-11-27T11:21:56.376515Z"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def crag(question, max_retries=3):\n",
    "    inputs = {\"keys\": {\"question\": question}}\n",
    "    while max_retries > 0:\n",
    "        try:\n",
    "            output = app.invoke(inputs)\n",
    "            return output['keys']['generation']\n",
    "        except Exception as e:\n",
    "            max_retries -= 1\n",
    "            print(f\"error: {e}, sleeping 2s, {max_retries} retries left\")\n",
    "            time.sleep(2)\n",
    "    # 使用之前的流程兜底\n",
    "    return rag(question, llm)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "166392d8-f801-4372-b8ad-3e79aef0b350",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:21:57.174926Z",
     "iopub.status.busy": "2024-11-27T11:21:57.174145Z",
     "iopub.status.idle": "2024-11-27T11:21:57.181518Z",
     "shell.execute_reply": "2024-11-27T11:21:57.181188Z",
     "shell.execute_reply.started": "2024-11-27T11:21:57.174855Z"
    },
    "papermill": {
     "duration": 0.141864,
     "end_time": "2024-11-23T14:35:27.564409",
     "exception": false,
     "start_time": "2024-11-23T14:35:27.422545",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "prediction_df = qa_df[qa_df['dataset'] == 'test'][['uuid', 'question', 'qa_type', 'answer']].rename(columns={'answer': 'ref_answer'})\n",
    "\n",
    "def predict(prediction_df):\n",
    "    prediction_df = prediction_df.copy()\n",
    "    answer_dict = {}\n",
    "\n",
    "    for idx, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        uuid = row['uuid']\n",
    "        question = row['question']\n",
    "        answer = crag(question)\n",
    "        answer_dict[question] = {\n",
    "            'uuid': uuid,\n",
    "            'ref_answer': row['ref_answer'],\n",
    "            'gen_answer': answer\n",
    "        }\n",
    "        \n",
    "    prediction_df.loc[:, 'gen_answer'] = prediction_df['question'].apply(lambda q: answer_dict[q]['gen_answer'])\n",
    "\n",
    "    return prediction_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca46d5f1-e698-457d-abb6-92d83cd59c66",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T11:30:35.876478Z",
     "iopub.status.busy": "2024-11-27T11:30:35.876242Z"
    },
    "papermill": {
     "duration": 514.92352,
     "end_time": "2024-11-23T14:44:02.805529",
     "exception": false,
     "start_time": "2024-11-23T14:35:27.882009",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "004337e4e4bd49c28989b5b05c4fd58c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: GENERATE---\n",
      "---GENERATE---\n",
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: GENERATE---\n",
      "---GENERATE---\n",
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: GENERATE---\n",
      "---GENERATE---\n",
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: GENERATE---\n",
      "---GENERATE---\n",
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: GENERATE---\n",
      "---GENERATE---\n",
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: GENERATE---\n",
      "---GENERATE---\n",
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT NOT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: TRANSFORM QUERY and RUN WEB SEARCH---\n",
      "---TRANSFORM QUERY---\n",
      "---WEB SEARCH---\n",
      "question: How can I contact the research team that wrote the report?\n",
      "---GENERATE---\n",
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: GENERATE---\n",
      "---GENERATE---\n",
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---DECIDE TO GENERATE---\n",
      "---DECISION: GENERATE---\n",
      "---GENERATE---\n",
      "---RETRIEVE---\n",
      "---CHECK RELEVANCE---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "---GRADE: DOCUMENT RELEVANT---\n",
      "error: list index out of range, sleeping 2s, 2 retries left\n"
     ]
    }
   ],
   "source": [
    "pred_df = predict(prediction_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4a1c867e-d59c-454d-93f5-2c8e23b666de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T14:11:12.393803Z",
     "iopub.status.busy": "2024-11-27T14:11:12.393518Z",
     "iopub.status.idle": "2024-11-27T14:11:12.530887Z",
     "shell.execute_reply": "2024-11-27T14:11:12.530261Z",
     "shell.execute_reply.started": "2024-11-27T14:11:12.393779Z"
    },
    "papermill": {
     "duration": 0.325117,
     "end_time": "2024-11-23T14:44:03.281713",
     "exception": false,
     "start_time": "2024-11-23T14:44:02.956596",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "save_path = os.path.join(expr_dir, 'pred_df.xlsx')\n",
    "pred_df.to_excel(save_path, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d79e974-089f-4c08-ba5e-804f6542e06a",
   "metadata": {
    "papermill": {
     "duration": 0.14423,
     "end_time": "2024-11-23T14:44:03.513124",
     "exception": false,
     "start_time": "2024-11-23T14:44:03.368894",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "217568fe-c0e4-49eb-9a7c-9fdfbc033d8a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T14:11:13.574956Z",
     "iopub.status.busy": "2024-11-27T14:11:13.574156Z",
     "iopub.status.idle": "2024-11-27T14:11:13.601120Z",
     "shell.execute_reply": "2024-11-27T14:11:13.600629Z",
     "shell.execute_reply.started": "2024-11-27T14:11:13.574864Z"
    },
    "papermill": {
     "duration": 0.369729,
     "end_time": "2024-11-23T14:44:04.017198",
     "exception": false,
     "start_time": "2024-11-23T14:44:03.647469",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "import time\n",
    "\n",
    "judge_llm = ChatOpenAI(\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    model_name='qwen2-72b-instruct',\n",
    "    temperature=0\n",
    ")\n",
    "\n",
    "def evaluate(prediction_df):\n",
    "    \"\"\"\n",
    "    对预测结果进行打分\n",
    "    :param prediction_df: 预测结果，需要包含问题，参考答案，生成的答案，列名分别为question, ref_answer, gen_answer\n",
    "    :return 打分模型原始返回结果\n",
    "    \"\"\"\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个经济学博士，现在我有一系列问题，有一个助手已经对这些问题进行了回答，你需要参照参考答案，评价这个助手的回答是否正确，仅回复“是”或“否”即可，不要带其他描述性内容或无关信息。\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "参考答案：\n",
    "<ref_answer>\n",
    "{{ref_answer}}\n",
    "</ref_answer>\n",
    "\n",
    "助手回答：\n",
    "<gen_answer>\n",
    "{{gen_answer}}\n",
    "</gen_answer>\n",
    "请评价：\n",
    "    \"\"\"\n",
    "    results = []\n",
    "\n",
    "    for _, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        question = row['question']\n",
    "        ref_answer = row['ref_answer']\n",
    "        gen_answer = row['gen_answer']\n",
    "\n",
    "        if not isinstance(gen_answer, str):\n",
    "            gen_answer = gen_answer.content\n",
    "        gen_answer = str(gen_answer)\n",
    "        \n",
    "        prompt = prompt_tmpl.replace('{{question}}', question).replace('{{ref_answer}}', str(ref_answer)).replace('{{gen_answer}}', gen_answer).strip()\n",
    "        \n",
    "        retry_count = 3\n",
    "        result = ''\n",
    "        \n",
    "        while retry_count > 0:\n",
    "            try:\n",
    "                result = judge_llm.invoke(prompt).content\n",
    "                break\n",
    "            except Exception as e:\n",
    "                retry_count -= 1\n",
    "                sleeping_seconds = 2 ** (4 - retry_count)\n",
    "                print(f\"query={question}, error={e}, sleeping={sleeping_seconds}, remaining retry count={retry_count}\")\n",
    "                \n",
    "                time.sleep(sleeping_seconds)\n",
    "        \n",
    "        results.append(result)\n",
    "\n",
    "        time.sleep(1)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "71db81af-b8f9-47ba-958b-761896516605",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T14:11:44.349793Z",
     "iopub.status.busy": "2024-11-27T14:11:44.349222Z",
     "iopub.status.idle": "2024-11-27T14:14:18.979539Z",
     "shell.execute_reply": "2024-11-27T14:14:18.977332Z",
     "shell.execute_reply.started": "2024-11-27T14:11:44.349743Z"
    },
    "papermill": {
     "duration": 150.566109,
     "end_time": "2024-11-23T14:46:34.714324",
     "exception": false,
     "start_time": "2024-11-23T14:44:04.148215",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "460ff3b3598d41a5a4bad961a93fc39a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_df['raw_score'] = evaluate(pred_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7da1b98e-99aa-4e11-9297-91eac1c62493",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T14:15:14.173631Z",
     "iopub.status.busy": "2024-11-27T14:15:14.172866Z",
     "iopub.status.idle": "2024-11-27T14:15:14.179025Z",
     "shell.execute_reply": "2024-11-27T14:15:14.178625Z",
     "shell.execute_reply.started": "2024-11-27T14:15:14.173562Z"
    },
    "papermill": {
     "duration": 0.138037,
     "end_time": "2024-11-23T14:46:35.040595",
     "exception": false,
     "start_time": "2024-11-23T14:46:34.902558",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['是', '否',\n",
       "       '否\\n\\n实际上，美国金融危机风险指标并没有一个统一的英文缩写为ROFCI。常见的金融危机风险指标包括金融压力指数（Financial Stress Index, FSI）、脆弱性指标（Vulnerability Indicator, VI）等。ROFCI并不是一个广泛认可的缩写。'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df['raw_score'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "2c99c078-d294-40b8-b57b-31cfd7349c3e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T14:15:15.047056Z",
     "iopub.status.busy": "2024-11-27T14:15:15.046195Z",
     "iopub.status.idle": "2024-11-27T14:15:15.057515Z",
     "shell.execute_reply": "2024-11-27T14:15:15.056156Z",
     "shell.execute_reply.started": "2024-11-27T14:15:15.046916Z"
    },
    "papermill": {
     "duration": 0.107466,
     "end_time": "2024-11-23T14:46:35.243603",
     "exception": false,
     "start_time": "2024-11-23T14:46:35.136137",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pred_df['score'] = (pred_df['raw_score'] == '是').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "423897f2-786e-415b-a613-55a4359faf76",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T14:15:16.150351Z",
     "iopub.status.busy": "2024-11-27T14:15:16.150169Z",
     "iopub.status.idle": "2024-11-27T14:15:16.153648Z",
     "shell.execute_reply": "2024-11-27T14:15:16.153201Z",
     "shell.execute_reply.started": "2024-11-27T14:15:16.150336Z"
    },
    "papermill": {
     "duration": 0.094328,
     "end_time": "2024-11-23T14:46:35.431162",
     "exception": false,
     "start_time": "2024-11-23T14:46:35.336834",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.45"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df['score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "79325429-9cf1-4e2c-95ac-cb0c1a3b6156",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-27T14:15:17.683304Z",
     "iopub.status.busy": "2024-11-27T14:15:17.682466Z",
     "iopub.status.idle": "2024-11-27T14:15:17.814801Z",
     "shell.execute_reply": "2024-11-27T14:15:17.812492Z",
     "shell.execute_reply.started": "2024-11-27T14:15:17.683232Z"
    },
    "papermill": {
     "duration": 0.289336,
     "end_time": "2024-11-23T14:46:35.804651",
     "exception": false,
     "start_time": "2024-11-23T14:46:35.515315",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "save_path = os.path.join(expr_dir, 'eval_df.xlsx')\n",
    "pred_df.to_excel(save_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88fc7227-9c21-48da-b179-5070406eb113",
   "metadata": {
    "papermill": {
     "duration": 0.088622,
     "end_time": "2024-11-23T14:46:36.016801",
     "exception": false,
     "start_time": "2024-11-23T14:46:35.928179",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1058.563616,
   "end_time": "2024-11-23T14:46:37.625874",
   "environment_variables": {},
   "exception": null,
   "input_path": "13_contextual_embeddings.ipynb",
   "output_path": "run_13_contextual_embeddings.ipynb",
   "parameters": {},
   "start_time": "2024-11-23T14:28:59.062258",
   "version": "2.6.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0cd8c168767249f2a5fa412173f6e751": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_5ce1d1d9d86c40d9839877ff95734491",
       "max": 100,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_231702cf4d79477f9d5548665a1b18fe",
       "tabbable": null,
       "tooltip": null,
       "value": 100
      }
     },
     "2133bb8d85d34b8db112b4408ad60320": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "231702cf4d79477f9d5548665a1b18fe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "23b1ad9c0f9c46c888da66e85c90eb84": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "24e6eadc3dc940ecabf30dd1a3c6d1f3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_fa4bddf2c33241b5bf918054518f128f",
       "max": 52,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_edc33e82be8f41eba6a18a0ef074ab7a",
       "tabbable": null,
       "tooltip": null,
       "value": 52
      }
     },
     "2f60367b1c8941e2bf71661c33969ae8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3865f25c78aa46f29a25d807205281c3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3d0b06deaa654b989eece8cde06fa0f8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "3f8ceda83287475b97608e42f5f6782f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4881e496f1c84fe29ce9ebebaddfb3c2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bd096d5d219a467786a85cfe1613fedd",
        "IPY_MODEL_24e6eadc3dc940ecabf30dd1a3c6d1f3",
        "IPY_MODEL_bc2b8104b4244d8cacedeb95e800d91c"
       ],
       "layout": "IPY_MODEL_6b9a8e43c1c342dba500a14e7149b600",
       "tabbable": null,
       "tooltip": null
      }
     },
     "5ce1d1d9d86c40d9839877ff95734491": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5ddb08be5cc64c9ab40a1d62a21763a5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_86283159049d48b1adcfb2de2d404d4d",
       "placeholder": "​",
       "style": "IPY_MODEL_2133bb8d85d34b8db112b4408ad60320",
       "tabbable": null,
       "tooltip": null,
       "value": " 100/100 [08:34&lt;00:00, 10.01s/it]"
      }
     },
     "5ef9d83ccad1471f85335900a24a8553": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6b9a8e43c1c342dba500a14e7149b600": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "816a079a8c804fbfa9b9a74f941abea8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_bcc69ec5db1b4aab977807284c9290e7",
        "IPY_MODEL_0cd8c168767249f2a5fa412173f6e751",
        "IPY_MODEL_5ddb08be5cc64c9ab40a1d62a21763a5"
       ],
       "layout": "IPY_MODEL_d1178c6858284f788a80b5f2a14fd0b7",
       "tabbable": null,
       "tooltip": null
      }
     },
     "86283159049d48b1adcfb2de2d404d4d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8ff8262c56604119883f4a5f13bb74ab": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_5ef9d83ccad1471f85335900a24a8553",
       "placeholder": "​",
       "style": "IPY_MODEL_e89e77133c344fc48c1d62f5a607ec93",
       "tabbable": null,
       "tooltip": null,
       "value": " 8/8 [00:18&lt;00:00,  2.27s/it]"
      }
     },
     "9189a076554543aaa6f5ee04e40dbe1b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "988e6697a2af486fadeaf0b84347b565": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_e1aae4c55cb64f379e74f15357275628",
        "IPY_MODEL_fd9e23198ca1489a9773fda3510bf857",
        "IPY_MODEL_8ff8262c56604119883f4a5f13bb74ab"
       ],
       "layout": "IPY_MODEL_d2ee15001d2244529f7e47d3333c0f8e",
       "tabbable": null,
       "tooltip": null
      }
     },
     "9fc7d91f94a94933bde5ba80e64587de": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "a7d240a289084bdfba4724c0efd5ab07": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "bc2b8104b4244d8cacedeb95e800d91c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_2f60367b1c8941e2bf71661c33969ae8",
       "placeholder": "​",
       "style": "IPY_MODEL_9fc7d91f94a94933bde5ba80e64587de",
       "tabbable": null,
       "tooltip": null,
       "value": " 52/52 [04:26&lt;00:00,  4.22s/it]"
      }
     },
     "bcc69ec5db1b4aab977807284c9290e7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_3f8ceda83287475b97608e42f5f6782f",
       "placeholder": "​",
       "style": "IPY_MODEL_3d0b06deaa654b989eece8cde06fa0f8",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "bd096d5d219a467786a85cfe1613fedd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_3865f25c78aa46f29a25d807205281c3",
       "placeholder": "​",
       "style": "IPY_MODEL_9189a076554543aaa6f5ee04e40dbe1b",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "cc3ed8dc4a5c43aca7b62d904865b2fa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cf68b6fe24964ce792aa63827489cb97": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "d1178c6858284f788a80b5f2a14fd0b7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d2ee15001d2244529f7e47d3333c0f8e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e1aae4c55cb64f379e74f15357275628": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_23b1ad9c0f9c46c888da66e85c90eb84",
       "placeholder": "​",
       "style": "IPY_MODEL_cf68b6fe24964ce792aa63827489cb97",
       "tabbable": null,
       "tooltip": null,
       "value": "100%"
      }
     },
     "e89e77133c344fc48c1d62f5a607ec93": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "edc33e82be8f41eba6a18a0ef074ab7a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "fa4bddf2c33241b5bf918054518f128f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "fd9e23198ca1489a9773fda3510bf857": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_cc3ed8dc4a5c43aca7b62d904865b2fa",
       "max": 8,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a7d240a289084bdfba4724c0efd5ab07",
       "tabbable": null,
       "tooltip": null,
       "value": 8
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
